diff --git a/.gitignore b/.gitignore index 891c0f87..d4acd86b 100644 --- a/.gitignore +++ b/.gitignore @@ -3,8 +3,17 @@ /output /prebuilts /cov +/deps +.autotools +.project +.cproject +.settings/ +/tests/frm/ *.ir *.out +*.DS_Store +.DS_Store +server_config.sh # Dynamic libraries # *.so diff --git a/build.sh b/build.sh old mode 100644 new mode 100755 diff --git a/ge/CMakeLists.txt b/ge/CMakeLists.txt index d10d73e4..6f54cf7b 100755 --- a/ge/CMakeLists.txt +++ b/ge/CMakeLists.txt @@ -10,34 +10,62 @@ elseif (ENABLE_D) add_subdirectory(ge_runtime) endif () -set(PROTO_LIST - "${METADEF_DIR}/proto/fusion_model.proto" - "${GE_CODE_DIR}/ge/proto/optimizer_priority.proto" -) - -set(PROTO_CLIENT_LIST - "${METADEF_DIR}/proto/ge_api.proto" -) - -set(PROTO_HEADER_LIST +set(GRAPHENGINE_PROTO_LIST "${METADEF_DIR}/proto/om.proto" "${METADEF_DIR}/proto/task.proto" "${METADEF_DIR}/proto/insert_op.proto" "${METADEF_DIR}/proto/ge_ir.proto" + "${METADEF_DIR}/proto/ge_api.proto" + "${METADEF_DIR}/proto/fusion_model.proto" + "${METADEF_DIR}/proto/optimizer_priority.proto" "${METADEF_DIR}/proto/fwk_adapter.proto" "${METADEF_DIR}/proto/op_mapping.proto" + "${METADEF_DIR}/proto/dump_task.proto" + "${METADEF_DIR}/proto/tensorflow/graph_library.proto" + "${METADEF_DIR}/proto/tensorflow/graph.proto" + "${METADEF_DIR}/proto/tensorflow/node_def.proto" + "${METADEF_DIR}/proto/tensorflow/function.proto" + "${METADEF_DIR}/proto/tensorflow/versions.proto" + "${METADEF_DIR}/proto/tensorflow/attr_value.proto" + "${METADEF_DIR}/proto/tensorflow/op_def.proto" + "${METADEF_DIR}/proto/tensorflow/tensor.proto" + "${METADEF_DIR}/proto/tensorflow/tensor_shape.proto" + "${METADEF_DIR}/proto/tensorflow/types.proto" + "${METADEF_DIR}/proto/tensorflow/resource_handle.proto" +) + +protobuf_generate(graphengine_protos GRAPHENGINE_PROTO_SRCS GRAPHENGINE_PROTO_HDRS ${GRAPHENGINE_PROTO_LIST} TARGET) + +set(GE_FUSION_MODEL_PROTO_SRCS + "${CMAKE_BINARY_DIR}/proto/graphengine_protos/proto/fusion_model.pb.cc" + "${CMAKE_BINARY_DIR}/proto/graphengine_protos/proto/optimizer_priority.pb.cc" ) -protobuf_generate(ge PROTO_SRCS PROTO_HDRS ${PROTO_LIST}) -protobuf_generate(ge PROTO_CLIENT_SRCS PROTO_CLIENT_HDRS ${PROTO_CLIENT_LIST}) -protobuf_generate(ge PROTO_HEADER_SRCS PROTO_HEADER_HDRS ${PROTO_HEADER_LIST}) -protobuf_generate(ge_client PROTO_CLIENT_HEADER_SRCS PROTO_CLIENT_HEADER_HDRS ${PROTO_HEADER_LIST}) + +add_library(ge_fusion_model_protos_obj OBJECT ${GE_FUSION_MODEL_PROTO_SRCS}) +add_dependencies(ge_fusion_model_protos_obj graphengine_protos) +target_include_directories(ge_fusion_model_protos_obj PRIVATE + #### blue zone #### + ${PROTOBUF_SHARED_PKG_DIR}/include + #### yellow zone #### + ${ASCEND_PROTOBUF_SHARED_PKG_DIR}/include +) +target_compile_definitions(ge_fusion_model_protos_obj PRIVATE + google=ascend_private +) +target_link_libraries(ge_fusion_model_protos_obj PRIVATE ascend_protobuf $) +target_compile_options(ge_fusion_model_protos_obj PRIVATE + $<$:-O2 -fPIC> + $<$,$>:-fexceptions> + $<$,$>: -Wno-deprecated-declarations -fno-common> + $<$,$>:/MTd> + $<$,$>:/MT> +) if (NOT ENABLE_D AND NOT ENABLE_ACL AND NOT ENABLE_MS_TESTCASES) ############ libge_proto_common.a ############ add_library(ge_proto_common STATIC - ${PROTO_HEADER_HDRS} - ${PROTO_SRCS} + $ ) target_compile_definitions(ge_proto_common PRIVATE @@ -57,8 +85,7 @@ target_link_libraries(ge_proto_common PRIVATE ############ libge_proto_client.a ############ add_library(ge_proto_client STATIC - ${PROTO_CLIENT_HEADER_HDRS} - ${PROTO_CLIENT_SRCS} + $ ) target_compile_definitions(ge_proto_client PRIVATE @@ -67,8 +94,8 @@ target_compile_definitions(ge_proto_client PRIVATE ) target_include_directories(ge_proto_client PRIVATE - ${CMAKE_BINARY_DIR}/proto/ge_client - ${CMAKE_BINARY_DIR}/proto/ge_client/proto + ${CMAKE_BINARY_DIR}/proto/graphengine_protos + ${CMAKE_BINARY_DIR}/proto/graphengine_protos/proto ) target_compile_options(ge_proto_client PRIVATE @@ -742,7 +769,7 @@ target_include_directories(ge_runner SYSTEM PRIVATE ${METADEF_DIR}/inc/external ${METADEF_DIR}/inc/graph ${CMAKE_BINARY_DIR} - ${CMAKE_BINARY_DIR}/proto/ge + ${CMAKE_BINARY_DIR}/proto/graphengine_protos #### yellow zone #### ${GE_CODE_DIR}/../inc ${GE_CODE_DIR}/../inc/external @@ -787,6 +814,10 @@ add_library(ge_compiler SHARED ${INFER_SRC_LIST} ) +add_dependencies(ge_compiler + graphengine_protos +) + target_compile_definitions(ge_compiler PRIVATE PROTOBUF_INLINE_NOT_IN_HEADERS=0 REUSE_MEMORY=1 @@ -819,7 +850,7 @@ target_include_directories(ge_compiler SYSTEM PRIVATE ${METADEF_DIR}/inc/external ${METADEF_DIR}/inc/graph ${CMAKE_BINARY_DIR} - ${CMAKE_BINARY_DIR}/proto/ge + ${CMAKE_BINARY_DIR}/proto/graphengine_protos #### yellow zone #### ${GE_CODE_DIR}/../inc ${GE_CODE_DIR}/../inc/external diff --git a/ge/client/ge_api.cc b/ge/client/ge_api.cc index 9cbd2d06..1aa4c41d 100644 --- a/ge/client/ge_api.cc +++ b/ge/client/ge_api.cc @@ -34,6 +34,7 @@ #include "common/ge/tbe_plugin_manager.h" #include "common/util/error_manager/error_manager.h" #include "toolchain/plog.h" +#include "ir_build/option_utils.h" using domi::OpRegistry; using std::map; @@ -69,16 +70,21 @@ Status CheckOptionsValid(const std::map &options) { auto job_id_iter = options.find(OPTION_EXEC_JOB_ID); if (job_id_iter != options.end()) { if (job_id_iter->second.length() > kMaxStrLen) { - GELOGE(PARAM_INVALID,"[Check][JobId]Failed," + GELOGE(PARAM_INVALID, "[Check][JobId]Failed," "the job_id [%s] string length: %zu > max string length: %d", job_id_iter->second.c_str(), job_id_iter->second.length(), kMaxStrLen); - REPORT_INPUT_ERROR("E10051", std::vector({"id","length"}), + REPORT_INPUT_ERROR("E10051", std::vector({"id", "length"}), std::vector({job_id_iter->second, std::to_string(kMaxStrLen)})); return FAILED; } } + // check modify_mixlist is valid + if (ge::CheckModifyMixlistParamValid(options) != ge::SUCCESS) { + return FAILED; + } + return SUCCESS; } @@ -244,7 +250,7 @@ std::string GEGetWarningMsg() { // Initialize session，which calls innerSession Session::Session(const std::map &options) { ErrorManager::GetInstance().SetStage(error_message::kInitialize, error_message::kOther); - GELOGT(TRACE_INIT, "Session Constructor start"); + GELOGT(TRACE_INIT, "Start to construct session."); ErrorManager::GetInstance().GenWorkStreamIdDefault(); // check init status @@ -332,7 +338,7 @@ Session::Session(const std::map &options) { // session destructor Session::~Session() { ErrorManager::GetInstance().SetStage(error_message::kFinalize, error_message::kFinalize); - GELOGT(TRACE_INIT, "Session Destructor start"); + GELOGT(TRACE_INIT, "Start to destruct session."); // 0.check init status if (!g_ge_initialized) { GELOGW("GE is not yet initialized or is finalized."); @@ -602,16 +608,16 @@ Status Session::RunGraph(uint32_t graph_id, const std::vector &inputs, s Status Session::RunGraphWithStreamAsync(uint32_t graph_id, void *stream, const std::vector &inputs, std::vector &outputs) { ErrorManager::GetInstance().SetStage(error_message::kModelCompile, error_message::kOther); - GELOGT(TRACE_INIT, "Session run graph with stream async start"); + GELOGT(TRACE_INIT, "Start to run graph with stream async."); ErrorManager::GetInstance().GenWorkStreamIdBySessionGraph(sessionId_, graph_id); std::shared_ptr instance_ptr = ge::GELib::GetInstance(); if (instance_ptr == nullptr) { GELOGE(GE_CLI_GE_NOT_INITIALIZED, - "[Run][Graph]Run graph with stream asyn failed, the GELib instance is nullptr," + "[Run][Graph]Run graph with stream async failed, the GELib instance is nullptr," "session id = %lu, graph id = %u, stream = %p.", sessionId_, graph_id, stream); REPORT_INNER_ERROR("E19999", - "Run graph with stream asyn failed, the GELib instance is nullptr" + "Run graph with stream async failed, the GELib instance is nullptr" "session id = %lu, graph id = %u, stream = %p.", sessionId_, graph_id, stream); return FAILED; } diff --git a/ge/common/CMakeLists.txt b/ge/common/CMakeLists.txt index 75cb8ad1..7974a46d 100755 --- a/ge/common/CMakeLists.txt +++ b/ge/common/CMakeLists.txt @@ -1,23 +1,3 @@ -set(PROTO_LIST - "${METADEF_DIR}/proto/om.proto" - "${METADEF_DIR}/proto/ge_ir.proto" - "${METADEF_DIR}/proto/insert_op.proto" - "${METADEF_DIR}/proto/task.proto" - "${METADEF_DIR}/proto/tensorflow/attr_value.proto" - "${METADEF_DIR}/proto/tensorflow/function.proto" - "${METADEF_DIR}/proto/tensorflow/graph.proto" - "${METADEF_DIR}/proto/tensorflow/node_def.proto" - "${METADEF_DIR}/proto/tensorflow/op_def.proto" - "${METADEF_DIR}/proto/tensorflow/resource_handle.proto" - "${METADEF_DIR}/proto/tensorflow/tensor.proto" - "${METADEF_DIR}/proto/tensorflow/tensor_shape.proto" - "${METADEF_DIR}/proto/tensorflow/types.proto" - "${METADEF_DIR}/proto/tensorflow/versions.proto" -) - -protobuf_generate(ge PROTO_SRCS PROTO_HDRS ${PROTO_LIST}) -protobuf_generate(ge_static PROTO_STATIC_SRCS PROTO_STATIC_HDRS ${PROTO_LIST}) - set(SRC_LIST "context/ctx.cc" "model_saver.cc" @@ -67,7 +47,12 @@ set(SRC_LIST if (NOT ENABLE_D AND NOT ENABLE_ACL) ############ libge_common.so ############ -add_library(ge_common SHARED ${SRC_LIST} ${PROTO_HDRS}) +add_library(ge_common SHARED ${SRC_LIST}) + +add_dependencies(ge_common + graphengine_protos +) + target_compile_definitions(ge_common PRIVATE PROTOBUF_INLINE_NOT_IN_HEADERS=0 HOST_VISIBILITY @@ -97,7 +82,7 @@ target_include_directories(ge_common PRIVATE ${METADEF_DIR}/inc/external/graph ${METADEF_DIR}/inc/graph ${CMAKE_BINARY_DIR} - ${CMAKE_BINARY_DIR}/proto/ge + ${CMAKE_BINARY_DIR}/proto/graphengine_protos #### yellow zone #### ${GE_DEPEND_DIR}/inc ${GE_DEPEND_DIR}/inc/cce @@ -128,7 +113,12 @@ target_link_libraries(ge_common PRIVATE ) ############ libge_common.a ############ -add_library(ge_common_static STATIC ${SRC_LIST} ${PROTO_STATIC_HDRS}) +add_library(ge_common_static STATIC ${SRC_LIST}) + +add_dependencies(ge_common_static + graphengine_protos +) + target_compile_definitions(ge_common_static PRIVATE PROTOBUF_INLINE_NOT_IN_HEADERS=0 HOST_VISIBILITY @@ -159,7 +149,7 @@ target_include_directories(ge_common_static PRIVATE ${METADEF_DIR}/inc/external/graph ${METADEF_DIR}/inc/graph ${CMAKE_BINARY_DIR} - ${CMAKE_BINARY_DIR}/proto/ge_static + ${CMAKE_BINARY_DIR}/proto/graphengine_protos #### yellow zone #### ${GE_DEPEND_DIR}/inc ${GE_DEPEND_DIR}/inc/cce @@ -180,7 +170,12 @@ target_link_libraries(ge_common_static PRIVATE else () ############ libge_common.so w/static protobuf ############ -add_library(ge_common SHARED ${SRC_LIST} ${PROTO_HDRS}) +add_library(ge_common SHARED ${SRC_LIST}) + +add_dependencies(ge_common + graphengine_protos +) + target_compile_definitions(ge_common PRIVATE PROTOBUF_INLINE_NOT_IN_HEADERS=0 HOST_VISIBILITY @@ -211,7 +206,7 @@ target_include_directories(ge_common PRIVATE ${METADEF_DIR}/inc/external/graph ${METADEF_DIR}/inc/graph ${CMAKE_BINARY_DIR} - ${CMAKE_BINARY_DIR}/proto/ge + ${CMAKE_BINARY_DIR}/proto/graphengine_protos ${GE_CODE_DIR}/third_party/fwkacllib/inc ${GE_CODE_DIR}/third_party/fwkacllib/inc/toolchain ) diff --git a/ge/common/dump/dump_op.cc b/ge/common/dump/dump_op.cc index e9414b2f..af603971 100755 --- a/ge/common/dump/dump_op.cc +++ b/ge/common/dump/dump_op.cc @@ -66,21 +66,21 @@ void DumpOp::SetDynamicModelInfo(const string &dynamic_model_name, const string static void SetOpMappingLoopAddr(uintptr_t step_id, uintptr_t loop_per_iter, uintptr_t loop_cond, toolkit::aicpu::dump::OpMappingInfo &op_mapping_info) { if (step_id != 0) { - GELOGI("step_id exists."); + GELOGI("Exists step_id."); op_mapping_info.set_step_id_addr(static_cast(step_id)); } else { GELOGI("step_id is null."); } if (loop_per_iter != 0) { - GELOGI("loop_per_iter exists."); + GELOGI("Exists loop_per_iter."); op_mapping_info.set_iterations_per_loop_addr(static_cast(loop_per_iter)); } else { GELOGI("loop_per_iter is null."); } if (loop_cond != 0) { - GELOGI("loop_cond exists."); + GELOGI("Exists loop_cond."); op_mapping_info.set_loop_cond_addr(static_cast(loop_cond)); } else { GELOGI("loop_cond is null."); @@ -253,7 +253,7 @@ Status DumpOp::LaunchDumpOp() { } if (device_id < 0) { GELOGE(ACL_ERROR_GE_INTERNAL_ERROR, "[Check][DeviceId]Failed, device_id %d", device_id); - REPORT_INNER_ERROR("E19999","Check device_id %d failed", device_id); + REPORT_INNER_ERROR("E19999", "Check device_id %d failed", device_id); return ACL_ERROR_GE_INTERNAL_ERROR; } toolkit::aicpu::dump::OpMappingInfo op_mapping_info; diff --git a/ge/common/formats/format_transfers/format_transfer_fracz_nchw.cc b/ge/common/formats/format_transfers/format_transfer_fracz_nchw.cc index c112aa79..aaeca490 100755 --- a/ge/common/formats/format_transfers/format_transfer_fracz_nchw.cc +++ b/ge/common/formats/format_transfers/format_transfer_fracz_nchw.cc @@ -72,8 +72,7 @@ Status CheckArgsForFracZToNchw(const TransArgs &args) { if (src_shape.at(kFracZHWC1) != dst_shape.at(kNchwH) * dst_shape.at(kNchwW) * c1 || src_shape.at(kFracZC0) != c0 || src_shape.at(kFracZNi) != kNiSize || src_shape.at(kFracZN0) != n0) { GELOGE(ACL_ERROR_GE_SHAPE_INVALID, - "[Check][Shape]Failed to check relationship between src and dst shape, " - "src shape %s, dst shape %s", + "[Check][Shape]Failed to check relationship between src and dst shape, src shape %s, dst shape %s", ShapeToString(src_shape).c_str(), ShapeToString(dst_shape).c_str()); REPORT_INNER_ERROR("E19999", "Failed to check relationship between src and dst shape, " "src shape %s, dst shape %s", @@ -138,9 +137,9 @@ Status GetDstDataAfterTrans(const TransArgs &args, TransResult &result, const in "[Operate][Memory]Failed to copy data from FracZ offset %ld to " "NCHW[%ld, %ld, %ld, %ld] offset %ld, err-code %d", src_offset, n_idx, c_idx, h_idx, w_idx, dst_offset, ret); - REPORT_CALL_ERROR("E19999","Failed to copy data from FracZ offset %ld to " + REPORT_CALL_ERROR("E19999", "Failed to copy data from FracZ offset %ld to " "NCHW[%ld, %ld, %ld, %ld] offset %ld, err-code %d", - src_offset, n_idx, c_idx, h_idx, w_idx, dst_offset, ret ); + src_offset, n_idx, c_idx, h_idx, w_idx, dst_offset, ret); return ACL_ERROR_GE_MEMORY_OPERATE_FAILED; } } diff --git a/ge/common/formats/format_transfers/format_transfer_fracz_nhwc.cc b/ge/common/formats/format_transfers/format_transfer_fracz_nhwc.cc index eb0d3801..1e71ea09 100755 --- a/ge/common/formats/format_transfers/format_transfer_fracz_nhwc.cc +++ b/ge/common/formats/format_transfers/format_transfer_fracz_nhwc.cc @@ -44,23 +44,20 @@ Status CheckArgsForFracZToNhwc(const TransArgs &args) { GELOGE(ACL_ERROR_GE_DATATYPE_INVALID, "[Check][DataType]Failed, " "shape from FORMAT_FRACTAL_Z to NCHW, invalid data type %s", TypeUtils::DataTypeToSerialString(args.src_data_type).c_str()); - REPORT_INNER_ERROR("E19999", "Failed to trans shape from FORMAT_FRACTAL_Z to NCHW, " - "invalid data type %s", + REPORT_INNER_ERROR("E19999", "Failed to trans shape from FORMAT_FRACTAL_Z to NCHW, invalid data type %s", TypeUtils::DataTypeToSerialString(args.src_data_type).c_str()); return ACL_ERROR_GE_DATATYPE_INVALID; } if (!CheckShapeValid(src_shape, kFracZDimsNum)) { GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "[Check][Shape]Value is invalid, src shape %s", ShapeToString(src_shape).c_str()); - REPORT_CALL_ERROR("E19999", "Src shape %s check invalid", - ShapeToString(src_shape).c_str()); + REPORT_CALL_ERROR("E19999", "Src shape %s check invalid", ShapeToString(src_shape).c_str()); return ACL_ERROR_GE_SHAPE_INVALID; } if (!CheckShapeValid(dst_shape, kNhwcDimsNum)) { GELOGE(ACL_ERROR_GE_SHAPE_INVALID, "[Check][Shape]Value is invalid, dst shape %s", ShapeToString(dst_shape).c_str()); - REPORT_CALL_ERROR("E19999", "Dst shape %s check invalid", - ShapeToString(dst_shape).c_str()); + REPORT_CALL_ERROR("E19999", "Dst shape %s check invalid", ShapeToString(dst_shape).c_str()); return ACL_ERROR_GE_SHAPE_INVALID; } int64_t c0 = GetCubeSizeByDataType(args.src_data_type); @@ -138,7 +135,7 @@ Status GetDstDataAfterTrans(const TransArgs &args, TransResult &result, int size "[Operate][Memory]Failed to copy data from FracZ offset %ld to " "NCHW[%ld, %ld, %ld, %ld] offset %ld, err-code %d", src_offset, n_idx, c_idx, h_idx, w_idx, dst_offset, ret); - REPORT_CALL_ERROR("E19999","Failed to copy data from FracZ offset %ld to " + REPORT_CALL_ERROR("E19999", "Failed to copy data from FracZ offset %ld to " "NCHW[%ld, %ld, %ld, %ld] offset %ld, err-code %d", src_offset, n_idx, c_idx, h_idx, w_idx, dst_offset, ret); return ACL_ERROR_GE_MEMORY_OPERATE_FAILED; @@ -185,7 +182,7 @@ Status FormatTransferFracZNhwc::TransFormat(const TransArgs &args, TransResult & ShapeToString(args.src_shape).c_str(), TypeUtils::DataTypeToSerialString(args.src_data_type).c_str(), ShapeToString(args.dst_shape).c_str(), total_size, ret); - REPORT_CALL_ERROR("E19999","Failed to get data after trans, src shape %s, data type %s, " + REPORT_CALL_ERROR("E19999", "Failed to get data after trans, src shape %s, data type %s, " "dst shape %s, memory size %ld, error_code %u", ShapeToString(args.src_shape).c_str(), TypeUtils::DataTypeToSerialString(args.src_data_type).c_str(), diff --git a/ge/common/formats/format_transfers/format_transfer_hwcn_c1hwncoc0.cc b/ge/common/formats/format_transfers/format_transfer_hwcn_c1hwncoc0.cc index 1e7f4f19..cb7f889b 100755 --- a/ge/common/formats/format_transfers/format_transfer_hwcn_c1hwncoc0.cc +++ b/ge/common/formats/format_transfers/format_transfer_hwcn_c1hwncoc0.cc @@ -112,11 +112,10 @@ Status GetDstDataAfterTrans(const TransArgs &args, TransResult &result, const in total_size, ShapeToString(args.dst_shape).c_str(), TypeUtils::FormatToSerialString(args.src_format).c_str(), TypeUtils::FormatToSerialString(args.dst_format).c_str()); - REPORT_CALL_ERROR("E19999", "Failed to alloc the memory for dst buf %ld, " - "shape %s when trans format from %s to %s", - total_size, ShapeToString(args.dst_shape).c_str(), - TypeUtils::FormatToSerialString(args.src_format).c_str(), - TypeUtils::FormatToSerialString(args.dst_format).c_str()); + REPORT_CALL_ERROR("E19999", "Failed to alloc the memory for dst buf %ld, shape %s when trans format from %s to %s", + total_size, ShapeToString(args.dst_shape).c_str(), + TypeUtils::FormatToSerialString(args.src_format).c_str(), + TypeUtils::FormatToSerialString(args.dst_format).c_str()); return ACL_ERROR_GE_MEMORY_ALLOCATION; } diff --git a/ge/common/formats/formats.cc b/ge/common/formats/formats.cc index 0c72a898..9e97a4d2 100755 --- a/ge/common/formats/formats.cc +++ b/ge/common/formats/formats.cc @@ -47,7 +47,7 @@ GE_FUNC_DEV_VISIBILITY GE_FUNC_HOST_VISIBILITY Status TransFormat(const TransArg GELOGE(ACL_ERROR_GE_PARAM_INVALID, "[Check][Shape]Failed, input data is null " "or shape size not euqal to 0, src_shape %s", ShapeToString(args.src_shape).c_str()); - REPORT_CALL_ERROR("E19999","Failed to check shape, input data is null " + REPORT_CALL_ERROR("E19999", "Failed to check shape, input data is null " "or shape size not equal to 0, src_shape %s", ShapeToString(args.src_shape).c_str()); return ACL_ERROR_GE_PARAM_INVALID; diff --git a/ge/common/helper/model_helper.cc b/ge/common/helper/model_helper.cc index ff32f123..45443c17 100644 --- a/ge/common/helper/model_helper.cc +++ b/ge/common/helper/model_helper.cc @@ -79,7 +79,8 @@ Status ModelHelper::SaveModelPartition(std::shared_ptr &om_fil Status ModelHelper::SaveSizeToModelDef(const GeModelPtr &ge_model) { vector om_info; auto ge_model_weight = ge_model->GetWeight(); - GELOGD("SaveSizeToModelDef weight_data_size is %zu, %p", ge_model_weight.GetSize(), ge_model_weight.GetData()); + GELOGD("SaveSizeToModelDef weight_data_size is %zu, ge_model_weight data is %p", ge_model_weight.GetSize(), + ge_model_weight.GetData()); om_info.push_back(ge_model_weight.GetSize()); TBEKernelStore tbe_kernel_store = ge_model->GetTBEKernelStore(); @@ -284,7 +285,7 @@ Status ModelHelper::SaveAllModelPartiton(std::shared_ptr& om_f if (SaveModelWeights(om_file_save_helper, ge_model, model_index) != SUCCESS) { GELOGE(FAILED, "[Save][ModelWeights]Failed, model %s, model index %zu", ge_model->GetName().c_str(), model_index); - REPORT_CALL_ERROR("E19999","ModelHelper save mode weights failed, model %s, model index %zu", + REPORT_CALL_ERROR("E19999", "ModelHelper save mode weights failed, model %s, model index %zu", ge_model->GetName().c_str(), model_index); return FAILED; } @@ -441,7 +442,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ModelHelper::SaveToOmRoo GELOGE(INTERNAL_ERROR, "[Save][AllModelPartition]Failed, model name %s, cur_index %zu", model_name.c_str(), cur_index); REPORT_CALL_ERROR("E19999", "Save all model %s partition failed, cur_index %zu", - model_name.c_str(), cur_index); + model_name.c_str(), cur_index); return INTERNAL_ERROR; } } @@ -459,7 +460,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ModelHelper::SaveToOmRoo GELOGE(FAILED, "[Save][Model]OmFileSaveHelper save model eturn fail, output_file %s", output_file.c_str()); REPORT_CALL_ERROR("E19999", "OmFileSaveHelper save model return fail, output_file %s", - output_file.c_str()); + output_file.c_str()); return FAILED; } return SUCCESS; @@ -601,7 +602,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ModelHelper::LoadModel(c FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ModelHelper::LoadRootModel(const ge::ModelData &model_data) { if (model_data.model_data == nullptr || model_data.model_len == 0) { GELOGE(ACL_ERROR_GE_EXEC_MODEL_DATA_SIZE_INVALID, "[Load][RootModel] " - "Model_data is nullptr or model_data_size is 0"); + "Model_data is nullptr or model data is empty."); REPORT_INNER_ERROR("E19999", "Load root model failed, model_data is nullptr or its size is 0"); return ACL_ERROR_GE_EXEC_MODEL_DATA_SIZE_INVALID; } @@ -628,7 +629,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ModelHelper::LoadRootMod //model verison 1.0 file header does not have model_num member is_unknown_shape_model_ = file_header_->version >= ge::MODEL_VERSION && file_header_->model_num > kStatiOmFileModelNum; - GELOGD("cur om model is ge root model or no %d, model version %u", is_unknown_shape_model_, file_header_->version); + GELOGD("Cur om model is ge root model or no %d, model version %u", is_unknown_shape_model_, file_header_->version); OmFileLoadHelper om_load_helper; if (is_unknown_shape_model_) { @@ -650,7 +651,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ModelHelper::LoadRootMod GELOGE(status, "[Generate][GERootModel]Failed"); return status; } - GELOGD("in ModelHelper::LoadRootModel, is_assign_model_ is setted to true!"); + GELOGD("In ModelHelper::LoadRootModel, is_assign_model_ is setted to true!"); is_assign_model_ = true; return SUCCESS; } @@ -790,7 +791,7 @@ Status ModelHelper::LoadWeights(OmFileLoadHelper &om_load_helper) { if (om_load_helper.GetModelPartition(ModelPartitionType::WEIGHTS_DATA, partition) != SUCCESS) { GELOGE(FAILED, "[Get][ModelWeightPartition]Failed, GetWeight size:%u", partition.size); REPORT_CALL_ERROR("E19999", "[Get][ModelPartition]Failed, GetWeight size:%u", - partition.size); + partition.size); return FAILED; } ge::Buffer weight = ge::Buffer::CopyFrom(partition.data, partition.size); @@ -805,7 +806,7 @@ Status ModelHelper::LoadWeights(OmFileLoadHelper &om_load_helper, GeModelPtr &cu if (om_load_helper.GetModelPartition(ModelPartitionType::WEIGHTS_DATA, partition, mode_index) != SUCCESS) { GELOGE(FAILED, "[Get][ModelPartition]Failed, GetWeight size:%u", partition.size); REPORT_CALL_ERROR("E19999", "[Get][ModelPartition]Failed, GetWeight size:%u", - partition.size); + partition.size); return FAILED; } ge::Buffer weight = ge::Buffer::CopyFrom(partition.data, partition.size); diff --git a/ge/common/model_parser/model_parser.cc b/ge/common/model_parser/model_parser.cc index ce654887..5c68eea8 100644 --- a/ge/common/model_parser/model_parser.cc +++ b/ge/common/model_parser/model_parser.cc @@ -62,7 +62,6 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ModelParserBase::LoadFro char *data = new (std::nothrow) char[len]; if (data == nullptr) { - GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "Load model From file failed, bad memory allocation occur. (need:%u)", len); GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "[Load][ModelFromFile]Failed, " "bad memory allocation occur(need %u), file %s", len, model_path); REPORT_CALL_ERROR("E19999", "Load model from file %s failed, " @@ -90,33 +89,45 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ModelParserBase::ParseMo GE_CHECK_NOTNULL(model.model_data); // Model length too small - GE_CHK_BOOL_RET_STATUS(model.model_len >= sizeof(ModelFileHeader), ACL_ERROR_GE_EXEC_MODEL_DATA_SIZE_INVALID, - "Invalid model. Model data size %u must be greater than or equal to %zu.", model.model_len, - sizeof(ModelFileHeader)); + GE_CHK_BOOL_EXEC(model.model_len >= sizeof(ModelFileHeader), + REPORT_INPUT_ERROR("E10003", std::vector({"parameter", "value", "reason"}), + std::vector({"om", model.om_name.c_str(), "invalid om file"})); + GELOGE(ACL_ERROR_GE_EXEC_MODEL_DATA_SIZE_INVALID, + "[Check][Param] Invalid model. Model data size %u must be greater than or equal to %zu.", + model.model_len, sizeof(ModelFileHeader)); + return ACL_ERROR_GE_EXEC_MODEL_DATA_SIZE_INVALID;); // Get file header auto file_header = reinterpret_cast(model.model_data); // Determine whether the file length and magic number match - GE_CHK_BOOL_RET_STATUS( - file_header->length == model.model_len - sizeof(ModelFileHeader) && file_header->magic == MODEL_FILE_MAGIC_NUM, - ACL_ERROR_GE_EXEC_MODEL_DATA_SIZE_INVALID, - "Invalid model. file_header->length[%u] + sizeof(ModelFileHeader)[%zu] != model->model_len[%u] || " - "MODEL_FILE_MAGIC_NUM[%u] != file_header->magic[%u]", - file_header->length, sizeof(ModelFileHeader), model.model_len, MODEL_FILE_MAGIC_NUM, file_header->magic); - + GE_CHK_BOOL_EXEC(file_header->length == model.model_len - sizeof(ModelFileHeader) && + file_header->magic == MODEL_FILE_MAGIC_NUM, + REPORT_INPUT_ERROR("E10003", std::vector({"parameter", "value", "reason"}), + std::vector({"om", model.om_name.c_str(), "invalid om file"})); + GELOGE(ACL_ERROR_GE_EXEC_MODEL_DATA_SIZE_INVALID, + "[Check][Param] Invalid model, file_header->length[%u] + sizeof(ModelFileHeader)[%zu] != " + "model->model_len[%u] || MODEL_FILE_MAGIC_NUM[%u] != file_header->magic[%u]", + file_header->length, sizeof(ModelFileHeader), model.model_len, + MODEL_FILE_MAGIC_NUM, file_header->magic); + return ACL_ERROR_GE_EXEC_MODEL_DATA_SIZE_INVALID;); Status res = SUCCESS; // Get data address uint8_t *data = reinterpret_cast(model.model_data) + sizeof(ModelFileHeader); if (file_header->is_encrypt == ModelEncryptType::UNENCRYPTED) { // Unencrypted model - GE_CHK_BOOL_RET_STATUS(model.key.empty(), ACL_ERROR_GE_PARAM_INVALID, - "Invalid param. model is unencrypted, but key is not empty."); - + if (!model.key.empty()) { + REPORT_INPUT_ERROR("E10003", std::vector({"parameter", "value", "reason"}), + std::vector({"om", model.om_name.c_str(), "invalid om file"})); + GELOGE(ACL_ERROR_GE_PARAM_INVALID, + "[Check][Param] Invalid param, model is unencrypted, but key is not empty."); + return ACL_ERROR_GE_PARAM_INVALID; + } model_data = data; model_len = file_header->length; GELOGD("Model_len is %u, model_file_head_len is %zu.", model_len, sizeof(ModelFileHeader)); } else { GELOGE(ACL_ERROR_GE_PARAM_INVALID, "[Check][Param]Invalid, model encrypt type not supported"); - REPORT_CALL_ERROR("E19999","Invalid model, encrypt type not supported"); + REPORT_INPUT_ERROR("E10003", std::vector({"parameter", "value", "reason"}), + std::vector({"om", model.om_name.c_str(), "invalid om file"})); res = ACL_ERROR_GE_PARAM_INVALID; } diff --git a/ge/common/op/ge_op_utils.cc b/ge/common/op/ge_op_utils.cc index ee5f6d34..99b5733c 100644 --- a/ge/common/op/ge_op_utils.cc +++ b/ge/common/op/ge_op_utils.cc @@ -444,17 +444,16 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status OpUtils::GetShapeDataFromConstTensor(const ConstGeTensorPtr &tensor, DataType type, std::vector &dims) { if (tensor == nullptr) { GELOGE(PARAM_INVALID, "[Check][Param]Input tensor is nullptr"); - REPORT_INNER_ERROR("E19999","Input tensor is nullptr"); + REPORT_INNER_ERROR("E19999", "Input tensor is nullptr"); return PARAM_INVALID; } // If the tensor data is a vector, the shape dimension must be 1 if (tensor->GetTensorDesc().GetShape().GetDims().size() > 1) { - GELOGE(PARAM_INVALID, "[Check][Param]The dimension of the input tensor shape " - "cannot be more than 1, it is %zu", + GELOGE(PARAM_INVALID, "[Check][Param]The dimension of the input tensor shape cannot be more than 1, it is %zu", tensor->GetTensorDesc().GetShape().GetDims().size()); - REPORT_CALL_ERROR("E19999", "The dimension of the input tensor shape %zu invalid, " - "more than 1", tensor->GetTensorDesc().GetShape().GetDims().size()); + REPORT_CALL_ERROR("E19999", "The dimension of the input tensor shape %zu invalid, more than 1", + tensor->GetTensorDesc().GetShape().GetDims().size()); return PARAM_INVALID; } @@ -473,8 +472,8 @@ OpUtils::GetShapeDataFromConstTensor(const ConstGeTensorPtr &tensor, DataType ty dims.push_back(shape_data[i]); } } else { - GELOGE(PARAM_INVALID, "[Check][DataType]Invalid, type only can be DT_INT32 or DT_INT64, " - "type is %s", TypeUtils::DataTypeToSerialString(type).c_str()); + GELOGE(PARAM_INVALID, "[Check][DataType]Invalid, type only can be DT_INT32 or DT_INT64, type is %s", + TypeUtils::DataTypeToSerialString(type).c_str()); REPORT_INNER_ERROR("E19999", "Data type %s check invalid, only can be DT_INT32 or DT_INT64", TypeUtils::DataTypeToSerialString(type).c_str()); return PARAM_INVALID; diff --git a/ge/common/profiling/profiling_manager.cc b/ge/common/profiling/profiling_manager.cc index f7015525..d615187f 100644 --- a/ge/common/profiling/profiling_manager.cc +++ b/ge/common/profiling/profiling_manager.cc @@ -184,7 +184,10 @@ ge::Status ProfilingManager::ParseOptions(const std::string &options) { if (options.find(kTrainingTrace) == std::string::npos) { return ge::SUCCESS; } - const std::string training_trace = prof_options[kTrainingTrace]; + std::string training_trace; + if (prof_options.contains(kTrainingTrace)) { + training_trace = prof_options[kTrainingTrace]; + } if (training_trace.empty()) { GELOGI("Training trace will not take effect."); return ge::SUCCESS; @@ -196,8 +199,12 @@ ge::Status ProfilingManager::ParseOptions(const std::string &options) { REPORT_INNER_ERROR("E19999", "Training trace param:%s is invalid.", training_trace.c_str()); return ge::PARAM_INVALID; } - fp_point_ = prof_options[kFpPoint]; - bp_point_ = prof_options[kBpPoint]; + if (prof_options.contains(kFpPoint)) { + fp_point_ = prof_options[kFpPoint]; + } + if (prof_options.contains(kBpPoint)) { + bp_point_ = prof_options[kBpPoint]; + } if (!fp_point_.empty() && !bp_point_.empty()) { GELOGI("Training trace bp fp is set, bp_point:%s, fp_point:%s.", bp_point_.c_str(), fp_point_.c_str()); } @@ -316,11 +323,14 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::Profilin FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfileStepInfo( uint64_t index_id, uint64_t model_id, uint16_t tag_id, rtStream_t stream, int32_t device_id) { #ifdef DAVINCI_SUPPORT_PROFILING - rtError_t rt_ret = RT_ERROR_NONE; -#ifndef ONLY_COMPILE_OPEN_SRC + if (!is_load_profiling_ && subscribe_count_ == 0) { + GELOGD("Profiling is not turned on, no need to profile step info."); + return SUCCESS; + } + GELOGD("Profiling Step Info TraceTask execute async start, index_id = %lu, model_id = %lu, tag_id = %u", index_id, model_id, tag_id); - rt_ret = rtProfilerTraceEx(index_id, model_id, tag_id, stream); + rtError_t rt_ret = rtProfilerTraceEx(index_id, model_id, tag_id, stream); if (rt_ret != RT_ERROR_NONE) { GELOGE(RT_FAILED, "[Call][rtProfilerTraceEx]Failed, ret 0x%X", rt_ret); REPORT_CALL_ERROR("E19999", "Call rtProfilerTraceEx failed, ret 0x%X", rt_ret); @@ -328,7 +338,6 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::Profil } GELOGD("Profiling Step Info TraceTask execute async success, index_id = %lu, model_id = %lu, tag_id = %u", index_id, model_id, tag_id); -#endif mmTimespec timespec = mmGetTickCount(); // 1000 ^ 3 converts second to nanosecond @@ -1014,10 +1023,12 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::GetFpBpP if (is_profiling_valid) { try { Json prof_options = Json::parse(profiling_options); - - fp_point_ = prof_options[kFpPoint]; - bp_point_ = prof_options[kBpPoint]; - + if (prof_options.contains(kFpPoint)) { + fp_point_ = prof_options[kFpPoint]; + } + if (prof_options.contains(kBpPoint)) { + bp_point_ = prof_options[kBpPoint]; + } fp_point = fp_point_; bp_point = bp_point_; if (!fp_point_.empty() && !bp_point_.empty()) { diff --git a/ge/engine_manager/dnnengine_manager.cc b/ge/engine_manager/dnnengine_manager.cc index 7a3f7aec..a2e39146 100644 --- a/ge/engine_manager/dnnengine_manager.cc +++ b/ge/engine_manager/dnnengine_manager.cc @@ -304,7 +304,7 @@ std::string DNNEngineManager::GetHostCpuEngineName(const std::vector &op GELOGE(FAILED, "[Get][HostCpuEngineName]Failed, HostCpuEngine not support [%s, %s]", op_desc->GetName().c_str(), op_desc->GetType().c_str()); REPORT_INNER_ERROR("E19999", "Get HostCpuEngineName failed, HostCpuEngine not support [%s, %s]", - op_desc->GetName().c_str(), op_desc->GetType().c_str()); + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return ""; } @@ -436,7 +436,7 @@ Status DNNEngineManager::ParserEngineMessage(const json engines_json, const std: GELOGE(FAILED, "[Check][Param]There are the same engine %s message in the json file", engine_id.c_str()); REPORT_INNER_ERROR("E19999", "There are the same engine %s message in the json file", - engine_id.c_str()); + engine_id.c_str()); return FAILED; } engines.emplace(engine_id, engine_conf_ptr); diff --git a/ge/executor/CMakeLists.txt b/ge/executor/CMakeLists.txt index 820518ad..d7da8916 100644 --- a/ge/executor/CMakeLists.txt +++ b/ge/executor/CMakeLists.txt @@ -1,15 +1,3 @@ -set(PROTO_LIST - "${METADEF_DIR}/proto/om.proto" - "${METADEF_DIR}/proto/ge_ir.proto" - "${METADEF_DIR}/proto/insert_op.proto" - "${METADEF_DIR}/proto/task.proto" - "${METADEF_DIR}/proto/op_mapping.proto" - "${METADEF_DIR}/proto/dump_task.proto" -) - -protobuf_generate(ge PROTO_SRCS PROTO_HDRS ${PROTO_LIST}) -protobuf_generate(ge_static PROTO_STATIC_SRCS PROTO_STATIC_HDRS ${PROTO_LIST}) - set(SRC_LIST "ge_executor.cc" "../common/profiling/profiling_manager.cc" @@ -19,7 +7,6 @@ set(SRC_LIST "../common/dump/exception_dumper.cc" "../common/dump/dump_manager.cc" "../common/dump/dump_op.cc" - "../common/dump/dump_server.cc" "../common/dump/opdebug_register.cc" "../common/profiling/ge_profiling.cc" "../graph/load/graph_loader.cc" @@ -168,7 +155,11 @@ set(SRC_LIST ) ######## libge_executor.a ######## -add_library(ge_executor STATIC ${SRC_LIST} ${PROTO_STATIC_HDRS}) +add_library(ge_executor STATIC ${SRC_LIST}) + +add_dependencies(ge_executor + graphengine_protos +) target_compile_options(ge_executor PRIVATE $<$,$>:-fvisibility=hidden -O2 -Werror -Wno-deprecated-declarations -fno-common> @@ -198,7 +189,7 @@ target_include_directories(ge_executor SYSTEM PRIVATE ${METADEF_DIR}/inc/external/graph ${METADEF_DIR}/inc/graph ${CMAKE_BINARY_DIR} - ${CMAKE_BINARY_DIR}/proto/ge_static + ${CMAKE_BINARY_DIR}/proto/graphengine_protos #### yellow zone #### ${GE_CODE_DIR}/../inc ${GE_CODE_DIR}/../inc/cce @@ -217,7 +208,11 @@ target_link_libraries(ge_executor PRIVATE ) ######## libge_executor.so ######## -add_library(ge_executor_shared SHARED ${SRC_LIST} ${PROTO_HDRS}) +add_library(ge_executor_shared SHARED ${SRC_LIST}) + +add_dependencies(ge_executor_shared + graphengine_protos +) target_compile_options(ge_executor_shared PRIVATE -fno-common @@ -245,13 +240,12 @@ target_include_directories(ge_executor_shared PRIVATE ${METADEF_DIR}/inc/external/graph ${METADEF_DIR}/inc/graph ${CMAKE_BINARY_DIR} - ${CMAKE_BINARY_DIR}/proto/ge + ${CMAKE_BINARY_DIR}/proto/graphengine_protos #### yellow zone #### ${GE_CODE_DIR}/../inc ${GE_CODE_DIR}/../inc/cce #### blue zone #### ${GE_CODE_DIR}/third_party/fwkacllib/inc - ${GE_CODE_DIR}/third_party/fwkacllib/inc/toolchain ) target_link_options(ge_executor_shared PRIVATE diff --git a/ge/ge_local_engine/CMakeLists.txt b/ge/ge_local_engine/CMakeLists.txt index ab767ccb..3675d333 100755 --- a/ge/ge_local_engine/CMakeLists.txt +++ b/ge/ge_local_engine/CMakeLists.txt @@ -1,7 +1,3 @@ -set(PROTO_LIST - "${METADEF_DIR}/proto/task.proto" -) - set(SRC_LIST "engine/ge_local_engine.cc" "ops_kernel_store/ge_local_ops_kernel_info.cc" @@ -19,12 +15,12 @@ set(OPS_KERNEL_SRC_LIST "ops_kernel_store/op/no_op.cc" ) -protobuf_generate(ge PROTO_SRCS PROTO_HDRS ${PROTO_LIST}) -protobuf_generate(ge_ops_shared PROTO_OPS_SHARED_SRCS PROTO_OPS_SHARED_HDRS ${PROTO_LIST}) -protobuf_generate(ge_ops_static PROTO_OPS_STATIC_SRCS PROTO_OPS_STATIC_HDRS ${PROTO_LIST}) - ############ libge_local_engine.so ############ -add_library(ge_local_engine SHARED ${SRC_LIST} ${PROTO_HDRS}) +add_library(ge_local_engine SHARED ${SRC_LIST}) + +add_dependencies(ge_local_engine + graphengine_protos +) target_compile_options(ge_local_engine PRIVATE -Werror @@ -48,7 +44,7 @@ target_include_directories(ge_local_engine PRIVATE ${METADEF_DIR}/inc/external/graph ${METADEF_DIR}/inc/graph ${CMAKE_BINARY_DIR} - ${CMAKE_BINARY_DIR}/proto/ge + ${CMAKE_BINARY_DIR}/proto/graphengine_protos #### yellow zone #### ${GE_CODE_DIR}/../inc #### blue zone #### @@ -70,7 +66,11 @@ target_link_libraries(ge_local_engine PRIVATE ) ######### atclib/libge_local_engine.so ############# -add_library(atc_ge_local_engine SHARED ${SRC_LIST} ${PROTO_HDRS}) +add_library(atc_ge_local_engine SHARED ${SRC_LIST}) + +add_dependencies(atc_ge_local_engine + graphengine_protos +) target_compile_options(atc_ge_local_engine PRIVATE -Werror @@ -94,7 +94,7 @@ target_include_directories(atc_ge_local_engine PRIVATE ${METADEF_DIR}/inc/external/graph ${METADEF_DIR}/inc/graph ${CMAKE_BINARY_DIR} - ${CMAKE_BINARY_DIR}/proto/ge + ${CMAKE_BINARY_DIR}/proto/graphengine_protos #### yellow zone #### ${GE_CODE_DIR}/../inc #### blue zone #### @@ -121,7 +121,11 @@ set_target_properties(atc_ge_local_engine PROPERTIES ) ############ libge_local_opskernel_builder.so ############ -add_library(ge_local_opskernel_builder SHARED ${OPS_KERNEL_SRC_LIST} ${PROTO_OPS_SHARED_HDRS}) +add_library(ge_local_opskernel_builder SHARED ${OPS_KERNEL_SRC_LIST}) + +add_dependencies(ge_local_opskernel_builder + graphengine_protos +) target_compile_options(ge_local_opskernel_builder PRIVATE -Werror @@ -145,7 +149,7 @@ target_include_directories(ge_local_opskernel_builder PRIVATE ${METADEF_DIR}/inc/external/graph ${METADEF_DIR}/inc/graph ${CMAKE_BINARY_DIR} - ${CMAKE_BINARY_DIR}/proto/ge_ops_shared + ${CMAKE_BINARY_DIR}/proto/graphengine_protos #### yellow zone #### ${GE_CODE_DIR}/../inc #### blue zone #### @@ -168,7 +172,11 @@ target_link_libraries(ge_local_opskernel_builder PRIVATE ) ############ atclib/libge_local_opskernel_builder.so ############ -add_library(atc_ge_local_opskernel_builder SHARED ${OPS_KERNEL_SRC_LIST} ${PROTO_OPS_SHARED_HDRS}) +add_library(atc_ge_local_opskernel_builder SHARED ${OPS_KERNEL_SRC_LIST}) + +add_dependencies(atc_ge_local_opskernel_builder + graphengine_protos +) target_compile_options(atc_ge_local_opskernel_builder PRIVATE -Werror @@ -192,7 +200,7 @@ target_include_directories(atc_ge_local_opskernel_builder PRIVATE ${METADEF_DIR}/inc/external/graph ${METADEF_DIR}/inc/graph ${CMAKE_BINARY_DIR} - ${CMAKE_BINARY_DIR}/proto/ge_ops_shared + ${CMAKE_BINARY_DIR}/proto/graphengine_protos #### yellow zone #### ${GE_CODE_DIR}/../inc #### blue zone #### @@ -220,7 +228,11 @@ set_target_properties(atc_ge_local_opskernel_builder PROPERTIES ) ############ libge_local_opskernel_builder.a ############ -add_library(ge_local_opskernel_builder_static STATIC ${OPS_KERNEL_SRC_LIST} ${PROTO_OPS_STATIC_HDRS}) +add_library(ge_local_opskernel_builder_static STATIC ${OPS_KERNEL_SRC_LIST}) + +add_dependencies(ge_local_opskernel_builder_static + graphengine_protos +) target_compile_options(ge_local_opskernel_builder_static PRIVATE -Werror @@ -245,7 +257,7 @@ target_include_directories(ge_local_opskernel_builder_static PRIVATE ${METADEF_DIR}/inc/external/graph ${METADEF_DIR}/inc/graph ${CMAKE_BINARY_DIR} - ${CMAKE_BINARY_DIR}/proto/ge_ops_static + ${CMAKE_BINARY_DIR}/proto/graphengine_protos #### yellow zone #### ${GE_CODE_DIR}/../inc #### blue zone #### diff --git a/ge/ge_local_engine/engine/host_cpu_engine.cc b/ge/ge_local_engine/engine/host_cpu_engine.cc index 8f01a166..f3a14317 100755 --- a/ge/ge_local_engine/engine/host_cpu_engine.cc +++ b/ge/ge_local_engine/engine/host_cpu_engine.cc @@ -26,12 +26,15 @@ #include "common/math/math_util.h" namespace { -#define CREATE_OUTPUT_CASE(DTYPE, TYPE) \ +#define CREATE_OUTPUT_CASE(DTYPE) \ case (DTYPE): { \ GeTensorPtr ge_tensor = nullptr; \ if (need_create_flag) { \ - uint64_t size = data_num * sizeof(TYPE); \ - ge_tensor = MakeShared(out_desc, size); \ + int64_t size = ge::GetSizeInBytes(static_cast(data_num), DTYPE); \ + if (size < 0) { \ + return INTERNAL_ERROR; \ + } \ + ge_tensor = MakeShared(out_desc, static_cast(size)); \ GE_CHECK_NOTNULL(ge_tensor); \ GELOGD("node:%s allocate output %zu success, size=%ld", op_desc->GetName().c_str(), i, size); \ ge_tensor->MutableTensorDesc().SetDataType(out_desc.GetDataType()); \ @@ -180,18 +183,19 @@ Status HostCpuEngine::PrepareOutputs(const ge::ConstOpDescPtr &op_desc, } } switch (out_desc.GetDataType()) { - CREATE_OUTPUT_CASE(DT_BOOL, bool) - CREATE_OUTPUT_CASE(DT_INT8, int8_t) - CREATE_OUTPUT_CASE(DT_INT16, int16_t) - CREATE_OUTPUT_CASE(DT_INT32, int32_t) - CREATE_OUTPUT_CASE(DT_INT64, int64_t) - CREATE_OUTPUT_CASE(DT_UINT8, uint8_t) - CREATE_OUTPUT_CASE(DT_UINT16, uint16_t) - CREATE_OUTPUT_CASE(DT_UINT32, uint32_t) - CREATE_OUTPUT_CASE(DT_UINT64, uint64_t) - CREATE_OUTPUT_CASE(DT_FLOAT16, fp16_t) - CREATE_OUTPUT_CASE(DT_FLOAT, float) - CREATE_OUTPUT_CASE(DT_DOUBLE, double) + CREATE_OUTPUT_CASE(DT_BOOL) + CREATE_OUTPUT_CASE(DT_INT8) + CREATE_OUTPUT_CASE(DT_INT16) + CREATE_OUTPUT_CASE(DT_INT32) + CREATE_OUTPUT_CASE(DT_INT64) + CREATE_OUTPUT_CASE(DT_UINT8) + CREATE_OUTPUT_CASE(DT_UINT16) + CREATE_OUTPUT_CASE(DT_UINT32) + CREATE_OUTPUT_CASE(DT_UINT64) + CREATE_OUTPUT_CASE(DT_FLOAT16) + CREATE_OUTPUT_CASE(DT_FLOAT) + CREATE_OUTPUT_CASE(DT_DOUBLE) + CREATE_OUTPUT_CASE(DT_INT4) default: GELOGW("data type %s not support.", TypeUtils::DataTypeToSerialString(out_desc.GetDataType()).c_str()); diff --git a/ge/generator/ge_generator.cc b/ge/generator/ge_generator.cc index a12f3cf7..8a94aa9b 100644 --- a/ge/generator/ge_generator.cc +++ b/ge/generator/ge_generator.cc @@ -31,6 +31,7 @@ #include "graph/ge_context.h" #include "graph/manager/graph_manager.h" #include "graph/manager/util/rt_context_util.h" +#include "graph/operator_factory_impl.h" #include "graph/opsproto_manager.h" #include "graph/utils/graph_utils.h" #include "graph/utils/type_utils.h" @@ -803,6 +804,41 @@ Status GeGenerator::CheckForSingleOp(OpDescPtr &op_desc, const vector return SUCCESS; } +Status GeGenerator::InferFormatForSingleOp(OpDescPtr &op_desc) { + GE_CHECK_NOTNULL(op_desc); + if (OperatorFactoryImpl::GetInferFormatFunc(op_desc->GetType()) != nullptr) { + auto node_op = ge::OperatorFactoryImpl::CreateOperator("node_op", op_desc->GetType()); + if (node_op.IsEmpty()) { + GELOGW("get op from OperatorFactory fail. op type: %s", op_desc->GetType().c_str()); + } else { + GELOGD("get op from OperatorFactory success. op type: %s", op_desc->GetType().c_str()); + auto temp_op_desc = ge::OpDescUtils::GetOpDescFromOperator(node_op); + if (temp_op_desc == nullptr) { + REPORT_INNER_ERROR("E19999", "GetOpDescFromOperator failed, as return nullptr, type:%s", + op_desc->GetType().c_str()); + GELOGE(FAILED, "[Get][OpDesc] temp op desc is null, type:%s", op_desc->GetType().c_str()); + return FAILED; + } + if (!op_desc->UpdateInputName(temp_op_desc->GetAllInputName())) { + GELOGW("InferFormatForSingleOp UpdateInputName failed"); + } + if (!op_desc->UpdateOutputName(temp_op_desc->GetAllOutputName())) { + GELOGW("InferFormatForSingleOp UpdateOutputName failed"); + } + } + node_op.BreakConnect(); + } + auto op = OpDescUtils::CreateOperatorFromOpDesc(op_desc); + auto ret = op_desc->CallInferFormatFunc(op); + if (ret != GRAPH_SUCCESS) { + REPORT_INNER_ERROR("E19999", "call InferFormatFunc for single op:%s fail", + op_desc->GetName().c_str()); + GELOGE(FAILED, "[Call][InferFormatFunc] for single op:%s fail.", op_desc->GetName().c_str()); + return FAILED; + } + return SUCCESS; +} + Status GeGenerator::BuildSingleOp(OpDescPtr &op_desc, const vector &inputs, const vector &outputs, const string &model_file_name, OpEngineType engine_type, ModelBufferData &model_buff, bool is_offline, int32_t compile_flag) { @@ -843,6 +879,7 @@ Status GeGenerator::BuildSingleOp(OpDescPtr &op_desc, const vector &in Graph graph; GE_CHK_STATUS(BuildSingleOpGraph(op_desc, inputs, outputs, name, graph), "[Build][Graph] for single op:%s fail.", op_desc->GetName().c_str()); + GE_CHK_STATUS_RET_NOLOG(InferFormatForSingleOp(op_desc)); // 2. check engine type when compile online if (model_file_name == kFileNameSuffix) { diff --git a/ge/graph/build/memory/graph_mem_assigner.cc b/ge/graph/build/memory/graph_mem_assigner.cc index 8becd90e..12790ec5 100755 --- a/ge/graph/build/memory/graph_mem_assigner.cc +++ b/ge/graph/build/memory/graph_mem_assigner.cc @@ -69,6 +69,10 @@ int64_t GetSymbolOutputOffset(const std::map &anchor_t } return ge::kInvalidOffset; } + +bool isVariableMemoryNode(const ge::NodePtr &node) { + return (node->GetType() == ge::VARIABLE) || (node->GetType() == ge::CONSTANTOP); +} } // namespace namespace ge { Status VariableMemoryAssigner::Assign() { @@ -447,22 +451,31 @@ bool IsContinuousInputConflict(const ge::NodePtr &node, const OpDescPtr &peer_op /// op1 -> node -> op2 /// return true when node is ref from input, and op1 or op2 is reuse input from output bool GraphMemoryAssigner::IsRefFromInputOpCascade(const NodePtr &node) { - bool ref_from_input = false; + std::unordered_set ref_input_index; int32_t reuse_in_index = -1; for (const auto &out_anchor : node->GetAllOutDataAnchors()) { - ref_from_input = GraphUtils::IsRefFromInput(out_anchor, reuse_in_index); - if (ref_from_input) { + bool reuse_input = GraphUtils::IsRefFromInput(out_anchor, reuse_in_index); + if (reuse_input) { GELOGD("IsRefFromInputOpCascade: cur node:%s:%d is ref", node->GetName().c_str(), reuse_in_index); - break; + ref_input_index.insert(reuse_in_index); } } + bool ref_from_input = !ref_input_index.empty(); + if (!ref_from_input) { + return false; + } for (const auto &in_anchor : node->GetAllInDataAnchors()) { const auto &peer_out_anchor = in_anchor->GetPeerOutAnchor(); GE_IF_BOOL_EXEC(peer_out_anchor == nullptr, continue); + auto in_node = peer_out_anchor->GetOwnerNode(); + if (isVariableMemoryNode(in_node) && (ref_input_index.count(in_anchor->GetIdx()) > 0)) { + GELOGD("Reuse variable memory, input node:%s, type:%s.", in_node->GetName().c_str(), in_node->GetType().c_str()); + return false; + } if (ref_from_input && GraphUtils::IsRefFromInput(peer_out_anchor, reuse_in_index)) { GELOGD("IsRefFromInputOpCascade: in node[%s] is ref, reuse index is:%d", - peer_out_anchor->GetOwnerNode()->GetName().c_str(), reuse_in_index); + in_node->GetName().c_str(), reuse_in_index); return true; } } @@ -500,6 +513,11 @@ Status GraphMemoryAssigner::UpdateRefOpOffsetReverse(const NodePtr &node) { GE_CHECK_NOTNULL(peer_out_anchor); auto peer_node = peer_out_anchor->GetOwnerNode(); GE_CHECK_NOTNULL(peer_node); + if (isVariableMemoryNode(peer_node)) { + GELOGW("Peer node to update is %s, skip it. Node name:%s.", + peer_node->GetType().c_str(), peer_node->GetName().c_str()); + continue; + } auto peer_op_desc = peer_node->GetOpDesc(); GE_CHECK_NOTNULL(peer_op_desc); vector peer_output_list = peer_op_desc->GetOutputOffset(); @@ -666,7 +684,8 @@ Status GraphMemoryAssigner::AssignContinuousInputMemory(const ge::NodePtr &node, bool is_allocated_first_input = is_continuous_input_allocated && (in_data_anchor->GetIdx() == 0); if (is_allocated_first_input) { std::map out2ins; - GE_CHK_STATUS_RET(TryGetNodeRefIndexes(node, out2ins), "[Get][RefIndexes]fail for node: %s", node->GetName().c_str()); + GE_CHK_STATUS_RET(TryGetNodeRefIndexes(node, out2ins), "[Get][RefIndexes]fail for node: %s", + node->GetName().c_str()); // output is beginning offset, set offset for input; only support this case now if ((out2ins.size() == 1) && (out2ins.begin()->second == 0) && (reverse_refresh)) { auto peer_output_offset = output_list.at(peer_out_data_anchor->GetIdx()); @@ -1496,6 +1515,12 @@ ge::Status GraphMemoryAssigner::UpdateOpInputOffset(const NodePtr &node, vector< output_list = last_peer_out_op_desc->GetOutputOffset(); auto out_index = static_cast(peer_out_anchor->GetIdx()); if (output_list.size() > static_cast(out_index)) { + int64_t peer_out_inner_offset = 0; + if (ge::AttrUtils::GetInt(last_peer_out_op_desc->MutableOutputDesc(out_index), ATTR_NAME_INNER_OFFSET, + peer_out_inner_offset)) { + (void)ge::AttrUtils::SetInt(tmp_op_desc->MutableInputDesc(anchor->GetIdx()), ATTR_NAME_INNER_OFFSET, + peer_out_inner_offset); + } bool is_l1_type = false; int64_t input_offset = output_list.at(out_index); if (has_mem_type_attr && !origin_input_list.empty()) { @@ -1510,15 +1535,27 @@ ge::Status GraphMemoryAssigner::UpdateOpInputOffset(const NodePtr &node, vector< GE_ERRORLOG_AND_ERRORMSG(ge::FAILED, error.c_str()); return ge::FAILED; } - GELOGD("Node[%s] input[%d] has origin offset[%ld]", tmp_op_desc->GetName().c_str(), anchor->GetIdx(), - origin_input_list[valid_input_index]); + int64_t inner_offset = 0; + (void)ge::AttrUtils::GetInt(tmp_op_desc->MutableInputDesc(anchor->GetIdx()), ATTR_NAME_INNER_OFFSET, + inner_offset); + GELOGD("Node[%s] input[%d] has origin offset[%ld] origin_inner_offset[%ld]", tmp_op_desc->GetName().c_str(), + anchor->GetIdx(), origin_input_list[valid_input_index], inner_offset); // L1 keep original input_offset is_l1_type = (memory_type[valid_input_index] == RT_MEMORY_L1); if (is_l1_type) { input_offset = origin_input_list[valid_input_index]; } else { // hbm input_offset = original input_offset + output_offset + if ((origin_input_list[valid_input_index] != 0) && (!tmp_op_desc->GetSubgraphInstanceNames().empty())) { + std::string error = "Node" + FmtToStr(tmp_op_desc->GetName()) + + +" has subgraphs which is conflict with has origin_input_list" + + FmtToStr(origin_input_list[valid_input_index]); + GE_ERRORLOG_AND_ERRORMSG(ge::FAILED, error.c_str()); + return ge::FAILED; + } input_offset = origin_input_list[valid_input_index] + output_list.at(out_index); + (void)ge::AttrUtils::SetInt(tmp_op_desc->MutableInputDesc(anchor->GetIdx()), ATTR_NAME_INNER_OFFSET, + origin_input_list[valid_input_index] + inner_offset); } } const auto &in_node = GetKnownInputNode(peer_out_anchor->GetOwnerNode()); @@ -1546,6 +1583,8 @@ ge::Status GraphMemoryAssigner::UpdateRefOpOutputOffset(const NodePtr &node, con const int ref_in, const int64_t input_offset) const { auto opdesc = node->GetOpDesc(); GE_CHECK_NOTNULL(opdesc); + int64_t inner_offset = 0; + bool has_inner_offset = ge::AttrUtils::GetInt(opdesc->MutableInputDesc(ref_in), ATTR_NAME_INNER_OFFSET, inner_offset); for (const auto &out2in : out2ins) { auto out_i = out2in.first; auto in_i = out2in.second; @@ -1559,8 +1598,11 @@ ge::Status GraphMemoryAssigner::UpdateRefOpOutputOffset(const NodePtr &node, con } origin_output_list[out_i] = input_offset; opdesc->SetOutputOffset(origin_output_list); - GELOGI("Node[%s] output[%d] is updated from reuse input index[%d] to offset[%ld]", opdesc->GetName().c_str(), - out_i, ref_in, input_offset); + if (has_inner_offset) { + (void)ge::AttrUtils::SetInt(opdesc->MutableOutputDesc(out_i), ATTR_NAME_INNER_OFFSET, inner_offset); + } + GELOGI("Node[%s] output[%d] is updated from reuse input index[%d] to offset[%ld], inner_offset[%ld]", opdesc->GetName().c_str(), + out_i, ref_in, input_offset, inner_offset); } } return ge::SUCCESS; diff --git a/ge/graph/build/model_builder.cc b/ge/graph/build/model_builder.cc index ce2f57f9..431e4882 100755 --- a/ge/graph/build/model_builder.cc +++ b/ge/graph/build/model_builder.cc @@ -246,7 +246,8 @@ Status ModelBuilder::SetInputOutputDesc() { } // if user set input node format ND, the expected node for data and netoutput format is ND in // final graph. - if ((compute_graph_->GetParentGraph() == nullptr) && (GetLocalOmgContext().format == domi::DOMI_TENSOR_ND) && (!node_op_desc->HasAttr("_is_single_op")) && + if ((compute_graph_->GetParentGraph() == nullptr) && (GetLocalOmgContext().format == domi::DOMI_TENSOR_ND) && + (!node_op_desc->HasAttr("_is_single_op")) && ((node_op_desc->GetType() == DATA_TYPE) || (node_op_desc->GetType() == NETOUTPUT))) { auto inputDescsPtr = node_op_desc->GetAllInputsDescPtr(); auto outputDescsPtr = node_op_desc->GetAllOutputsDescPtr(); diff --git a/ge/graph/build/stream_allocator.cc b/ge/graph/build/stream_allocator.cc index 0e1a1aba..a5a1112e 100644 --- a/ge/graph/build/stream_allocator.cc +++ b/ge/graph/build/stream_allocator.cc @@ -905,6 +905,7 @@ Status StreamAllocator::SplitStreams(vector> &split_streams) { added_stream_num_vec[stream_id]++; new_stream_id_vec[stream_id] = last_stream_id; split_streams[stream_id].emplace(last_stream_id); + split_ori_stream_map_[last_stream_id] = stream_id; node_split_stream_map_[cur_node] = last_stream_id; // Add the send/recv event to the first and last nodes of the split stream. @@ -1104,7 +1105,7 @@ Status StreamAllocator::UpdateActiveStreamsForActiveNode(const vector(new_split_stream)); active_streams.assign(new_active_streams.begin(), new_active_streams.end()); if (!AttrUtils::SetListInt(active_op, ATTR_NAME_ACTIVE_STREAM_LIST, active_streams)) { @@ -1148,13 +1150,21 @@ Status StreamAllocator::UpdateActiveStreamsForSubgraphs() const { } bool StreamAllocator::IsActivated(int64_t stream_id) const { + const auto &iter = split_ori_stream_map_.find(stream_id); + if (iter == split_ori_stream_map_.end()) { + REPORT_INNER_ERROR("E19999", "Find original stream_id failed, split_stream_id=%ld", stream_id); + GELOGE(INTERNAL_ERROR, "[CheckActivated][Check] Find original stream_id failed, split_stream_id=%ld", stream_id); + return false; + } + int64_t ori_stream_id = iter->second; for (const auto &node : whole_graph_->GetNodes(whole_graph_->GetGraphUnknownFlag())) { auto op_desc = node->GetOpDesc(); vector active_streams; if (op_desc == nullptr || !AttrUtils::GetListInt(op_desc, ATTR_NAME_ACTIVE_STREAM_LIST, active_streams)) { continue; } - if (std::find(active_streams.begin(), active_streams.end(), stream_id) != active_streams.end()) { + if (std::find(active_streams.begin(), active_streams.end(), stream_id) != active_streams.end() || + std::find(active_streams.begin(), active_streams.end(), ori_stream_id) != active_streams.end()) { return true; } } diff --git a/ge/graph/build/stream_allocator.h b/ge/graph/build/stream_allocator.h index 44dcd673..34b2ec3a 100644 --- a/ge/graph/build/stream_allocator.h +++ b/ge/graph/build/stream_allocator.h @@ -66,7 +66,7 @@ class StreamAllocator { Status UpdateActiveStreamsForSwitchNode(NodePtr &switch_node); Status InsertActiveNodesAfterSwitch(NodePtr &switch_nodes, std::vector &switch_active_nodes); Status UpdateActiveStreamsForActiveNode(const std::vector> &split_streams, NodePtr &node); - Status UpdateActiveStreamsForSubgraphs() const; + Status UpdateActiveStreamsForSubgraphs(); bool IsActivated(int64_t stream_id) const; Status SetActiveStreamsForLoop(); Status CheckStreamActived() const; @@ -114,6 +114,7 @@ class StreamAllocator { std::map> specific_activated_streams_nodes_map_; std::map node_split_stream_map_; + std::map split_ori_stream_map_; std::map subgraph_first_active_node_map_; // send events corresponding to the node @@ -123,4 +124,4 @@ class StreamAllocator { std::map> node_to_recv_events_; }; } // namespace ge -#endif // GE_GRAPH_BUILD_STREAM_ALLOCATOR_H_ \ No newline at end of file +#endif // GE_GRAPH_BUILD_STREAM_ALLOCATOR_H_ diff --git a/ge/graph/build/task_generator.cc b/ge/graph/build/task_generator.cc index dabdc5d2..2db0b6b7 100755 --- a/ge/graph/build/task_generator.cc +++ b/ge/graph/build/task_generator.cc @@ -187,33 +187,61 @@ Status TaskGenerator::AddModelTaskToModel(const ModelTaskDef &model_task_def, ui } Status TaskGenerator::UpdateOpIsVarAttr(const OpDescPtr &op_desc, uint64_t session_id) { - vector input_offsets = op_desc->GetInputOffset(); GELOGD("Update is var attr, node[name:%s(%s), id:%ld, stream_id:%ld].", op_desc->GetName().c_str(), op_desc->GetType().c_str(), op_desc->GetId(), op_desc->GetStreamId()); + // input + vector input_offsets = op_desc->GetInputOffset(); if (!(input_offsets.empty())) { vector input_var; - for (int64_t input : input_offsets) { - input_var.push_back(VarManager::Instance(session_id)->IsVarAddr(input)); + size_t valid_input_index = 0; + for (uint32_t i = 0; i < op_desc->GetAllInputsSize(); i++) { + vector output_list; + auto input_tensor_desc = op_desc->MutableInputDesc(i); + if (input_tensor_desc == nullptr) { + continue; + } + if (valid_input_index >= input_offsets.size()) { + break; + } + int64_t inner_offset = 0; + (void)ge::AttrUtils::GetInt(input_tensor_desc, ATTR_NAME_INNER_OFFSET, inner_offset); + GELOGD("Node[%s] input[%u] has inner_offset[%ld]", op_desc->GetName().c_str(), i, inner_offset); + input_var.push_back(VarManager::Instance(session_id)->IsVarAddr(input_offsets[valid_input_index] - inner_offset)); + valid_input_index++; } GE_CHK_BOOL_EXEC(AttrUtils::SetListBool(op_desc, kIsInputVar, input_var), REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", kIsInputVar, op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(FAILED, "[Set][Attr] %s fail for op:%s(%s)", kIsInputVar, - op_desc->GetName().c_str(), op_desc->GetType().c_str()); + GELOGE(FAILED, "[Set][Attr] %s fail for op:%s(%s)", kIsInputVar, op_desc->GetName().c_str(), + op_desc->GetType().c_str()); return FAILED); } - + // output vector output_offsets = op_desc->GetOutputOffset(); if (!(output_offsets.empty())) { vector output_var; - for (int64_t output : output_offsets) { - output_var.push_back(VarManager::Instance(session_id)->IsVarAddr(output)); + size_t valid_output_index = 0; + for (uint32_t i = 0; i < op_desc->GetAllOutputsDescSize(); i++) { + vector output_list; + auto output_tensor_desc = op_desc->MutableOutputDesc(i); + if (output_tensor_desc == nullptr) { + continue; + } + if (valid_output_index >= output_offsets.size()) { + break; + } + int64_t inner_offset = 0; + (void)ge::AttrUtils::GetInt(output_tensor_desc, ATTR_NAME_INNER_OFFSET, inner_offset); + GELOGD("Node[%s] output[%u] has inner_offset[%ld]", op_desc->GetName().c_str(), i, inner_offset); + output_var.push_back( + VarManager::Instance(session_id)->IsVarAddr(output_offsets[valid_output_index] - inner_offset)); + valid_output_index++; } GE_CHK_BOOL_EXEC(AttrUtils::SetListBool(op_desc, kIsOutputVar, output_var), REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", kIsOutputVar, op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(FAILED, "[Set][Attr] %s fail for op:%s(%s)", kIsOutputVar, - op_desc->GetName().c_str(), op_desc->GetType().c_str()); + GELOGE(FAILED, "[Set][Attr] %s fail for op:%s(%s)", kIsOutputVar, op_desc->GetName().c_str(), + op_desc->GetType().c_str()); return FAILED); } return SUCCESS; diff --git a/ge/graph/common/omg_util.cc b/ge/graph/common/omg_util.cc index 1dba8c51..598677bd 100644 --- a/ge/graph/common/omg_util.cc +++ b/ge/graph/common/omg_util.cc @@ -193,23 +193,29 @@ Status SetCyclicDependenceFlag(const ge::NodePtr &node) { /// /// @brief set op next_iteration name -/// @param [in] node -/// @param [in] next +/// @param [in] Merge Node +/// @param [in] NextIteration Node /// @return Status /// -Status SetNextIteration(const ge::NodePtr &node, const std::string &next) { +Status SetNextIteration(const NodePtr &node, const NodePtr &next) { GE_CHECK_NOTNULL(node); - OpDescPtr tmp_desc = node->GetOpDesc(); - GE_CHECK_NOTNULL(tmp_desc); + GE_CHECK_NOTNULL(next); + GE_CHECK_NOTNULL(node->GetOpDesc()); + GE_CHECK_NOTNULL(next->GetOpDesc()); - if (!AttrUtils::SetStr(tmp_desc, ge::ATTR_NAME_NEXT_ITERATION, next)) { - REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", ATTR_NAME_NEXT_ITERATION.c_str(), - node->GetName().c_str(), node->GetType().c_str()); - GELOGE(FAILED, "[Set][Attr] %s fail for op:%s(%s)", ATTR_NAME_NEXT_ITERATION.c_str(), - node->GetName().c_str(), node->GetType().c_str()); - return FAILED; - } + const auto SetIterationName = [](const OpDescPtr &op_desc, const std::string &name) { + if (!AttrUtils::SetStr(op_desc, ATTR_NAME_NEXT_ITERATION, name)) { + REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", ATTR_NAME_NEXT_ITERATION.c_str(), + op_desc->GetName().c_str(), op_desc->GetType().c_str()); + GELOGE(FAILED, "[Set][Attr] %s fail for op:%s(%s)", ATTR_NAME_NEXT_ITERATION.c_str(), + op_desc->GetName().c_str(), op_desc->GetType().c_str()); + return FAILED; + } + return SUCCESS; + }; + GE_CHK_STATUS_RET_NOLOG(SetIterationName(node->GetOpDesc(), next->GetName())); + GE_CHK_STATUS_RET_NOLOG(SetIterationName(next->GetOpDesc(), node->GetName())); return SUCCESS; } @@ -272,19 +278,24 @@ bool IsUnknownShapeTensor(const GeTensorDesc &tensor_desc) { /// @brief Set Op _force_unknown_shape flag /// @param [in] node /// @param [in] force_unknown, set attribute if true +/// @param [in] group_index, condition group index of node. /// @return /// -void MarkForceUnknownShape(const NodePtr &node, bool force_unknown) { - GE_RT_VOID_CHECK_NOTNULL(node); +void MarkForceUnknownShape(const NodePtr &node, bool force_unknown, int64_t group_index) { if (!force_unknown) { return; } - GELOGD("[%s] mark as force unknown shape node", node->GetName().c_str()); - if (!AttrUtils::SetBool(node->GetOpDesc(), ATTR_NAME_FORCE_UNKNOWN_SHAPE, force_unknown)) { - REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", ATTR_NAME_FORCE_UNKNOWN_SHAPE.c_str(), + GE_RT_VOID_CHECK_NOTNULL(node); + const auto &op_desc = node->GetOpDesc(); + GE_RT_VOID_CHECK_NOTNULL(op_desc); + + // op_desc as AttrHolderAdapter valid, Set attribute always success, just log for check. + GELOGD("[%s] Set control flow group index: %ld", node->GetName().c_str(), group_index); + if (!AttrUtils::SetInt(op_desc, ATTR_NAME_CONTROL_FLOW_GROUP, group_index)) { + REPORT_INNER_ERROR("E19999", "Set Attr:%s fail for op:%s(%s)", ATTR_NAME_CONTROL_FLOW_GROUP.c_str(), node->GetName().c_str(), node->GetType().c_str()); - GELOGE(FAILED, "[Set][Attr] %s fail for op:%s(%s)", ATTR_NAME_FORCE_UNKNOWN_SHAPE.c_str(), + GELOGE(FAILED, "[Set][Attr] %s fail for op:%s(%s)", ATTR_NAME_CONTROL_FLOW_GROUP.c_str(), node->GetName().c_str(), node->GetType().c_str()); } } diff --git a/ge/graph/common/omg_util.h b/ge/graph/common/omg_util.h index c84da7f8..91fcd29e 100644 --- a/ge/graph/common/omg_util.h +++ b/ge/graph/common/omg_util.h @@ -96,11 +96,11 @@ Status SetCyclicDependenceFlag(const ge::NodePtr &node); /// /// @brief set op next_iteration name -/// @param [in] node -/// @param [in] next +/// @param [in] Merge Node +/// @param [in] NextIteration Node /// @return Status /// -Status SetNextIteration(const ge::NodePtr &node, const std::string &next); +Status SetNextIteration(const NodePtr &node, const NodePtr &next); /// /// @brief Align the memory @@ -129,9 +129,10 @@ bool IsUnknownShapeTensor(const GeTensorDesc &tensor_desc); /// @brief Set Op _force_unknown_shape flag /// @param [in] node /// @param [in] force_unknown, set attribute if true +/// @param [in] group_index, condition group index of node. /// @return /// -void MarkForceUnknownShape(const NodePtr &node, bool force_unknown); +void MarkForceUnknownShape(const NodePtr &node, bool force_unknown, int64_t group_index); } // namespace ge #endif // GE_GRAPH_COMMON_OMG_UTIL_H_ diff --git a/ge/graph/execute/graph_execute.cc b/ge/graph/execute/graph_execute.cc index 87dadb41..02d7d3ca 100755 --- a/ge/graph/execute/graph_execute.cc +++ b/ge/graph/execute/graph_execute.cc @@ -704,7 +704,7 @@ Status GraphExecutor::GetCurShape(const uint32_t model_id, std::vector } Status GraphExecutor::GetOpAttr(uint32_t model_id, const std::string &op_name, const std::string &attr_name, - std::string &attr_value) { + std::string &attr_value) { auto model_manager = ge::ModelManager::GetInstance(); GE_CHECK_NOTNULL(model_manager); Status ret = model_manager->GetOpAttr(model_id, op_name, attr_name, attr_value); diff --git a/ge/graph/load/graph_loader.cc b/ge/graph/load/graph_loader.cc index e4904614..7ee74d1d 100755 --- a/ge/graph/load/graph_loader.cc +++ b/ge/graph/load/graph_loader.cc @@ -33,12 +33,12 @@ Status GraphLoader::UnloadModel(uint32_t model_id) { Status ret = model_manager->Stop(model_id); if (ret != SUCCESS) { - GELOGE(ret, "UnloadModel: Stop failed. model id:%u", model_id); + GELOGE(ret, "[Stop][Model] failed. model id:%u", model_id); } ret = model_manager->Unload(model_id); if (ret != SUCCESS) { - GELOGE(ret, "UnloadModel: Unload failed. model id:%u", model_id); + GELOGE(ret, "[Unload][Model] failed. model id:%u", model_id); return ret; } GELOGI("UnLoad model success, model id:%u.", model_id); @@ -50,14 +50,13 @@ Status GraphLoader::LoadModelOnline(uint32_t &model_id, const std::shared_ptrLoadModelOnline(model_id, ge_root_model_ptr, listener); if (ret != SUCCESS) { - GELOGE(ret, "LoadModel: Load failed. ret = %u", ret); + GELOGE(ret, "[Load][Model] Online failed. ret = %u, model_id:%u", ret, model_id); rt_ret = rtDeviceReset(GetContext().DeviceId()); if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtDeviceReset failed, device_id:%u, ret:0x%X", GetContext().DeviceId(), rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtDeviceReset] failed, device_id:%u, ret:0x%X", GetContext().DeviceId(), rt_ret); } return ret; } @@ -81,31 +80,31 @@ Status GraphLoader::LoadModelOnline(uint32_t &model_id, const std::shared_ptrStart(model_id); if (ret != SUCCESS) { if (model_manager->Unload(model_id) != SUCCESS) { - GELOGE(ret, "LoadModel: Unload failed while trying to unload after a failed start."); + GELOGE(ret, "[Unload][Model] failed while trying to unload after a failed start, model_id:%u.", model_id); } rt_ret = rtDeviceReset(GetContext().DeviceId()); if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtDeviceReset failed, device_id:%u, ret:0x%X", GetContext().DeviceId(), rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtDeviceReset] failed, device_id:%u, ret:0x%X", GetContext().DeviceId(), rt_ret); } - GELOGE(ret, "LoadModel: Start failed."); + GELOGE(ret, "[Start][Model] failed, model_id:%u.", model_id); return ret; } rt_ret = rtDeviceReset(GetContext().DeviceId()); if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtDeviceReset failed, device_id:%u, ret:0x%X", GetContext().DeviceId(), rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtDeviceReset] failed, device_id:%u, ret:0x%X", GetContext().DeviceId(), rt_ret); return RT_FAILED; } GELOGI("Load model online success, model_id:%u.", model_id); @@ -118,7 +117,7 @@ Status GraphLoader::GetMaxUsedMemory(uint32_t model_id, uint64_t &max_size) { GE_CHECK_NOTNULL(model_manager); Status ret = model_manager->GetMaxUsedMemory(model_id, max_size); if (ret != SUCCESS) { - GELOGE(ret, "GetMaxUsedMemory: GetMaxUsedMemory failed."); + GELOGE(ret, "[Call][GetMaxUsedMemory] failed, model_id:%u.", model_id); return ret; } return SUCCESS; @@ -127,21 +126,20 @@ Status GraphLoader::GetMaxUsedMemory(uint32_t model_id, uint64_t &max_size) { Status GraphLoader::LoadDataFromFile(const std::string &path, const std::string &key_path, int32_t priority, ModelData &model_data) { if (!CheckInputPathValid(path)) { - GELOGE(ACL_ERROR_GE_EXEC_MODEL_PATH_INVALID, "model path is invalid: %s", path.c_str()); + GELOGE(ACL_ERROR_GE_EXEC_MODEL_PATH_INVALID, "[Check][Param] model path is invalid:%s", path.c_str()); return ACL_ERROR_GE_EXEC_MODEL_PATH_INVALID; } GELOGI("Load model begin, model path is: %s", path.c_str()); if (!key_path.empty() && !CheckInputPathValid(key_path)) { - REPORT_INNER_ERROR("E19999", "Param key_path:%s empty or invalid", - key_path.c_str()); - GELOGE(ACL_ERROR_GE_PARAM_INVALID, "decrypt_key path is invalid: %s", key_path.c_str()); + REPORT_INNER_ERROR("E19999", "Param key_path:%s empty or invalid", key_path.c_str()); + GELOGE(ACL_ERROR_GE_PARAM_INVALID, "[Check][Param] decrypt_key path is invalid:%s", key_path.c_str()); return ACL_ERROR_GE_PARAM_INVALID; } Status ret = ModelParserBase::LoadFromFile(path.c_str(), key_path.c_str(), priority, model_data); if (ret != SUCCESS) { - GELOGE(ret, "LoadModelFromFile: Load failed. ret = %u", ret); + GELOGE(ret, "[Call][LoadFromFile] failed. ret = %u, path:%s, key path:%s", ret, path.c_str(), key_path.c_str()); if (model_data.model_data != nullptr) { delete[] static_cast(model_data.model_data); model_data.model_data = nullptr; @@ -156,18 +154,19 @@ Status GraphLoader::CommandHandle(const Command &command) { GE_CHECK_NOTNULL(model_manager); Status ret = model_manager->HandleCommand(command); if (ret != SUCCESS) { - GELOGE(ret, "CommandHandle: Command Handle failed."); + GELOGE(ret, "[Handle][Command] failed, module_index:%lu.", command.module_index); return ret; } } catch (std::bad_alloc &) { REPORT_INNER_ERROR("E19999", "Bad memory allocation occur"); - GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "Command handle failed, bad memory allocation occur !"); + GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "[Handle][Command] failed, " + "bad memory allocation occur, module_index:%lu.", command.module_index); return ACL_ERROR_GE_MEMORY_ALLOCATION; } catch (...) { REPORT_INNER_ERROR("E19999", "Some exceptions occur"); - GELOGE(FAILED, "Command handle failed, some exceptions occur !"); + GELOGE(FAILED, "[Handle][Command] failed, some exceptions occur, module_index:%lu.", command.module_index); return FAILED; } @@ -184,7 +183,7 @@ Status GraphLoader::LoadModelFromData(uint32_t &model_id, const ModelData &model Status ret = model_manager->LoadModelOffline( model_id, model_data, nullptr, dev_ptr, mem_size, weight_ptr, weight_size); if (ret != SUCCESS) { - GELOGE(ret, "Load model failed, model_id:%u.", model_id); + GELOGE(ret, "[Load][Model] failed, model_id:%u.", model_id); return ret; } GELOGI("Load model success, model_id:%u.", model_id); @@ -210,7 +209,7 @@ Status GraphLoader::LoadModelWithQ(uint32_t &model_id, const ModelData &model_da GE_CHECK_NOTNULL(model_manager); Status ret = model_manager->LoadModelWithQ(model_id, model_data, input_queue_ids, output_queue_ids); if (ret != SUCCESS) { - GELOGE(ret, "Load model with queue failed, model_id:%u.", model_id); + GELOGE(ret, "[Load][Model] with queue failed, model_id:%u.", model_id); return ret; } @@ -237,7 +236,7 @@ Status GraphLoader::ExecuteModel(uint32_t model_id, rtStream_t stream, bool asyn Status ret = model_manager->ExecuteModel(model_id, stream, async_mode, input_data, input_desc, output_data, output_desc); if (ret != SUCCESS) { - GELOGE(ret, "Execute model failed, model_id:%u.", model_id); + GELOGE(ret, "[Execute][Model] failed, model_id:%u.", model_id); return ret; } @@ -250,7 +249,7 @@ Status GraphLoader::GetMemoryInfo(int64_t &free) { if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtSetDevice failed, device_id:%u, ret:0x%X", GetContext().DeviceId(), rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtSetDevice] failed, device_id:%u, ret:0x%X", GetContext().DeviceId(), rt_ret); return RT_FAILED; } size_t total_mem = 0; @@ -258,14 +257,14 @@ Status GraphLoader::GetMemoryInfo(int64_t &free) { rt_ret = rtMemGetInfo(&free_mem, &total_mem); if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtMemGetInfo failed, ret:0x%X", rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtMemGetInfo] failed, ret:0x%X", rt_ret); return RT_FAILED; } rt_ret = rtDeviceReset(GetContext().DeviceId()); if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtDeviceReset failed, device_id:%u, ret:0x%X", GetContext().DeviceId(), rt_ret); - GELOGE(RT_FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtDeviceReset] failed, device_id:%u, ret:0x%X", GetContext().DeviceId(), rt_ret); return RT_FAILED; } // Add small page memory size @@ -280,7 +279,8 @@ Status GraphLoader::DestroyAicpuKernel(uint64_t session_id, uint32_t model_id, u GE_CHECK_NOTNULL(model_manager); Status ret = model_manager->DestroyAicpuKernel(session_id, model_id, sub_model_id); if (ret != SUCCESS) { - GELOGE(ret, "Destroy aicpu kernel failed."); + GELOGE(ret, "[Destroy][AicpuKernel] failed, session_id:%lu, model_id:%u, sub_model_id:%u.", + session_id, model_id, sub_model_id); return ret; } return SUCCESS; @@ -291,7 +291,7 @@ Status GraphLoader::DestroyAicpuSessionForInfer(uint32_t model_id) { GE_CHECK_NOTNULL(model_manager); Status ret = model_manager->DestroyAicpuSessionForInfer(model_id); if (ret != SUCCESS) { - GELOGE(ret, "Destroy aicpu serrion for infer failed."); + GELOGE(ret, "[Call][DestroyAicpuSessionForInfer] failed, model_id:%u.", model_id); return ret; } return SUCCESS; diff --git a/ge/graph/load/model_manager/davinci_model.cc b/ge/graph/load/model_manager/davinci_model.cc index b52796c8..f8b61216 100755 --- a/ge/graph/load/model_manager/davinci_model.cc +++ b/ge/graph/load/model_manager/davinci_model.cc @@ -488,6 +488,8 @@ void DavinciModel::InitRuntimeParams() { session_scope_mem_info.memory_size = static_cast(ret ? value : 0); runtime_param_.memory_infos[kSessionScopeMemory | RT_MEMORY_HBM] = std::move(session_scope_mem_info); + ret = ge::AttrUtils::GetInt(ge_model_, ATTR_MODEL_ZERO_COPY_MEMORY_SIZE, value); + runtime_param_.zero_copy_size = ret ? value : 0; GELOGI("InitRuntimeParams(), %s.", runtime_param_.ToString().c_str()); } @@ -3977,7 +3979,6 @@ Status DavinciModel::NnExecute(rtStream_t stream, bool async_mode, const InputDa is_dynamic_ = input_data.is_dynamic_batch; bool profiling_model_execute_on = ProfilingManager::Instance().ProfilingModelExecuteOn(); - bool profiling_model_load_on = ProfilingManager::Instance().ProfilingModelLoadOn(); GE_IF_BOOL_EXEC(profiling_model_execute_on, SetProfileTime(MODEL_PRE_PROC_START)); Status ret = CopyModelData(input_data, output_data, is_dynamic_); GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, return ret, @@ -3991,10 +3992,8 @@ Status DavinciModel::NnExecute(rtStream_t stream, bool async_mode, const InputDa uint64_t model_id = static_cast(model_id_); int32_t device_id = static_cast(device_id_); // tag_id 0 means step begin, 1 meas step end. - if (profiling_model_load_on) { - GE_CHK_STATUS_RET_NOLOG( - ProfilingManager::Instance().ProfileStepInfo(index_id, model_id, 0, rt_model_stream_, device_id)); - } + GE_CHK_STATUS_RET_NOLOG( + ProfilingManager::Instance().ProfileStepInfo(index_id, model_id, 0, rt_model_stream_, device_id)); GELOGD("rtModelExecute do"); GE_IF_BOOL_EXEC(profiling_model_execute_on, SetProfileTime(MODEL_INFER_START)); @@ -4003,10 +4002,8 @@ Status DavinciModel::NnExecute(rtStream_t stream, bool async_mode, const InputDa GE_IF_BOOL_EXEC(profiling_model_execute_on, SetProfileTime(MODEL_INFER_END)); GELOGD("rtModelExecute end"); - if (profiling_model_load_on) { - GE_CHK_STATUS_RET_NOLOG( - ProfilingManager::Instance().ProfileStepInfo(index_id, model_id, 1, rt_model_stream_, device_id)); - } + GE_CHK_STATUS_RET_NOLOG( + ProfilingManager::Instance().ProfileStepInfo(index_id, model_id, 1, rt_model_stream_, device_id)); iterator_count_++; } @@ -4505,4 +4502,22 @@ void DavinciModel::UpdateOpIOAddrs(uint32_t task_id, uint32_t stream_id, const s op_desc_info->output_addrs = output_addrs; GELOGD("[Update][OpIOAddrs] Op [%s] update input output addr success.", op_desc_info->op_name.c_str()); } + +/// +/// @ingroup ge +/// @brief Get total useful size, in known subgraph, no need to allocate zero copy memory during initialization. +/// @param [in] total_useful_size: total mem size - zero copy size. +/// @return Status +/// +Status DavinciModel::GetTotalMemSizeExcludeZeroCopy(int64_t &total_useful_size) { + if (runtime_param_.mem_size < static_cast(runtime_param_.zero_copy_size)) { + REPORT_CALL_ERROR("E19999", "total mem size[%lu] is less than zero copy size[%ld] ", runtime_param_.mem_size, + runtime_param_.zero_copy_size); + GELOGE(FAILED, "[Check][TotalMemSizeExcludeZeroCopy] failed, total mem size[%lu] is less than zero copy size[%ld]", + runtime_param_.mem_size, runtime_param_.zero_copy_size); + return FAILED; + } + total_useful_size = runtime_param_.mem_size - runtime_param_.zero_copy_size; + return SUCCESS; +} } // namespace ge diff --git a/ge/graph/load/model_manager/davinci_model.h b/ge/graph/load/model_manager/davinci_model.h index e4898dec..8a8fb35e 100755 --- a/ge/graph/load/model_manager/davinci_model.h +++ b/ge/graph/load/model_manager/davinci_model.h @@ -248,6 +248,14 @@ class DavinciModel { // get total mem size size_t TotalMemSize() const { return runtime_param_.mem_size; } + /// + /// @ingroup ge + /// @brief Get total useful size, in known subgraph, no need to allocate zero copy memory during initialization. + /// @param [in] total_useful_size: total mem size - zero copy size. + /// @return Status + /// + Status GetTotalMemSizeExcludeZeroCopy(int64_t &total_useful_size); + // model name string Name() const { return name_; } diff --git a/ge/graph/load/model_manager/model_manager.cc b/ge/graph/load/model_manager/model_manager.cc index 47d104f4..d7a6ca25 100755 --- a/ge/graph/load/model_manager/model_manager.cc +++ b/ge/graph/load/model_manager/model_manager.cc @@ -27,7 +27,6 @@ #include "graph/load/model_manager/davinci_model.h" #include "model/ge_root_model.h" #include "common/formats/utils/formats_trans_utils.h" -#include "toolchain/adx_datadump_server.h" namespace ge { thread_local uint32_t device_count = 0; @@ -49,7 +48,6 @@ const int kTimeSpecNano = 1000000000; const int kTimeSpecMiro = 1000000; const int kOpNameMaxSize = 100; const uint64_t kInferSessionId = 0; -const int32_t kDumpStatus = 0; #pragma pack(push, 1) struct CustAicpuSoBuf { uint64_t kernelSoBuf; @@ -102,16 +100,14 @@ Status ModelManager::KernelLaunchEx(aicpu::FWKAdapter::FWKOperateType op_type, u auto kernel_size = sizeof(uint64_t) * (v_aicpu_kernel.size()); rtError_t rt_ret = rtMalloc(&aicpu_kernel_addr, kernel_size, RT_MEMORY_HBM); GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, - REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%zu, ret: 0x%X", - kernel_size, rt_ret); - GELOGE(RT_FAILED, "rtMalloc error, ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%zu, ret:0x%X", kernel_size, rt_ret); + GELOGE(RT_FAILED, "[Call][RtMalloc] failed, size:%zu, ret:0x%X", kernel_size, rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret);) rt_ret = rtMemcpy(aicpu_kernel_addr, kernel_size, v_aicpu_kernel.data(), kernel_size, RT_MEMCPY_HOST_TO_DEVICE); GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, - REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%zu, ret: 0x%X", - kernel_size, rt_ret); - GELOGE(RT_FAILED, "rtMemcpy to input_output_addr_ error: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%zu, ret:0x%X", kernel_size, rt_ret); + GELOGE(RT_FAILED, "[Call][RtMemcpy] failed, size:%zu, ret:0x%X", kernel_size, rt_ret); GE_CHK_RT(rtFree(aicpu_kernel_addr)); return RT_ERROR_TO_GE_STATUS(rt_ret);) uint64_t kernel_id_addr = static_cast(reinterpret_cast(aicpu_kernel_addr)); param_base.fwkKernelBase.fwk_kernel.kernelID = kernel_id_addr; @@ -122,9 +118,8 @@ Status ModelManager::KernelLaunchEx(aicpu::FWKAdapter::FWKOperateType op_type, u rtError_t rt_ret = rtMalloc(&(devicebase), sizeof(STR_FWK_OP_KERNEL), RT_MEMORY_HBM); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%zu, ret: 0x%X", - sizeof(STR_FWK_OP_KERNEL), rt_ret); - GELOGE(RT_FAILED, "malloc device memory failed. ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%zu, ret:0x%X", sizeof(STR_FWK_OP_KERNEL), rt_ret); + GELOGE(RT_FAILED, "[Call][RtMalloc] failed. size:%zu, ret:0x%X", sizeof(STR_FWK_OP_KERNEL), rt_ret); GE_IF_BOOL_EXEC(aicpu_kernel_addr != nullptr, GE_CHK_RT(rtFree(aicpu_kernel_addr))); return RT_ERROR_TO_GE_STATUS(rt_ret); } @@ -132,9 +127,8 @@ Status ModelManager::KernelLaunchEx(aicpu::FWKAdapter::FWKOperateType op_type, u rt_ret = rtMemcpy(devicebase, sizeof(STR_FWK_OP_KERNEL), ¶m_base, sizeof(STR_FWK_OP_KERNEL), RT_MEMCPY_HOST_TO_DEVICE); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%zu, ret: 0x%X", - sizeof(STR_FWK_OP_KERNEL), rt_ret); - GELOGE(RT_FAILED, "memory copy to device failed. ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%zu, ret:0x%X", sizeof(STR_FWK_OP_KERNEL), rt_ret); + GELOGE(RT_FAILED, "[Call][RtMemcpy] failed, size:%zu, ret:0x%X", sizeof(STR_FWK_OP_KERNEL), rt_ret); GE_IF_BOOL_EXEC(aicpu_kernel_addr != nullptr, GE_CHK_RT(rtFree(aicpu_kernel_addr))); GE_CHK_RT(rtFree(devicebase)); return RT_ERROR_TO_GE_STATUS(rt_ret); @@ -143,8 +137,8 @@ Status ModelManager::KernelLaunchEx(aicpu::FWKAdapter::FWKOperateType op_type, u rtStream_t stream = nullptr; rt_ret = rtStreamCreate(&stream, 0); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtStreamCreate failed, ret: 0x%X", rt_ret); - GELOGE(RT_FAILED, "create stream failed. ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtStreamCreate failed, ret:0x%X", rt_ret); + GELOGE(RT_FAILED, "[Create][Stream] failed. ret:0x%X", rt_ret); GE_IF_BOOL_EXEC(aicpu_kernel_addr != nullptr, GE_CHK_RT(rtFree(aicpu_kernel_addr))); GE_CHK_RT(rtFree(devicebase)); return RT_ERROR_TO_GE_STATUS(rt_ret); @@ -152,8 +146,8 @@ Status ModelManager::KernelLaunchEx(aicpu::FWKAdapter::FWKOperateType op_type, u rt_ret = rtKernelLaunchEx(devicebase, sizeof(STR_FWK_OP_KERNEL), 0, stream); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtKernelLaunchEx failed, ret: 0x%X", rt_ret); - GELOGE(RT_FAILED, "rtKernelLaunchEx failed. ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtKernelLaunchEx failed, ret:0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtKernelLaunchEx] failed. ret:0x%X", rt_ret); GE_IF_BOOL_EXEC(aicpu_kernel_addr != nullptr, GE_CHK_RT(rtFree(aicpu_kernel_addr))); GE_CHK_RT(rtFree(devicebase)); GE_CHK_RT(rtStreamDestroy(stream)); @@ -161,9 +155,8 @@ Status ModelManager::KernelLaunchEx(aicpu::FWKAdapter::FWKOperateType op_type, u } rt_ret = rtStreamSynchronize(stream); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtStreamSynchronize failed, ret: 0x%X", - rt_ret); - GELOGE(RT_FAILED, "rtStreamSynchronize failed. ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtStreamSynchronize failed, ret:0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtStreamSynchronize] failed. ret:0x%X", rt_ret); GE_IF_BOOL_EXEC(aicpu_kernel_addr != nullptr, GE_CHK_RT(rtFree(aicpu_kernel_addr))); GE_CHK_RT(rtFree(devicebase)); GE_CHK_RT(rtStreamDestroy(stream)); @@ -172,8 +165,8 @@ Status ModelManager::KernelLaunchEx(aicpu::FWKAdapter::FWKOperateType op_type, u if (aicpu_kernel_addr != nullptr) { rt_ret = rtFree(aicpu_kernel_addr); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtFree failed, ret: 0x%X", rt_ret); - GELOGE(RT_FAILED, "free memory failed. ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtFree failed, ret:0x%X", rt_ret); + GELOGE(RT_FAILED, "[Free][Memory] failed. ret:0x%X", rt_ret); GE_CHK_RT(rtFree(devicebase)); GE_CHK_RT(rtStreamDestroy(stream)); return RT_ERROR_TO_GE_STATUS(rt_ret); @@ -181,15 +174,15 @@ Status ModelManager::KernelLaunchEx(aicpu::FWKAdapter::FWKOperateType op_type, u } rt_ret = rtFree(devicebase); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtFree failed, ret: 0x%X", rt_ret); - GELOGE(RT_FAILED, "free memory failed. ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtFree failed, ret:0x%X", rt_ret); + GELOGE(RT_FAILED, "[Free][Memory] failed. ret:0x%X", rt_ret); GE_CHK_RT(rtStreamDestroy(stream)); return RT_ERROR_TO_GE_STATUS(rt_ret); } rt_ret = rtStreamDestroy(stream); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtStreamDestroy failed, ret: 0x%X", rt_ret); - GELOGE(RT_FAILED, "rtStreamDestroy failed. ret: 0x%X", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtStreamDestroy failed, ret:0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtStreamDestroy] failed. ret:0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } return SUCCESS; @@ -235,9 +228,8 @@ ge::Status ModelManager::DestroyAicpuSessionForInfer(uint32_t model_id) { auto it = model_map_.find(model_id); if (it == model_map_.end()) { - REPORT_INNER_ERROR("E19999", "Param model_id:%u can't find in model_map, check invalid", - model_id); - GELOGE(ACL_ERROR_GE_EXEC_MODEL_ID_INVALID, "model id %u does not exists.", model_id); + REPORT_INNER_ERROR("E19999", "Param model_id:%u can't find in model_map, check invalid", model_id); + GELOGE(ACL_ERROR_GE_EXEC_MODEL_ID_INVALID, "[Check][Param] model id %u does not exists.", model_id); return ACL_ERROR_GE_EXEC_MODEL_ID_INVALID; } uint64_t session_id = it->second->GetSessionId(); @@ -256,7 +248,8 @@ ge::Status ModelManager::DestroyAicpuKernel(uint64_t session_id, uint32_t model_ if (ret != SUCCESS) { REPORT_CALL_ERROR("E19999", "Call KernelLaunchEx fail, model_id:%u, sub_model_id:%u, session_id:%lu", model_id, sub_model_id, session_id); - GELOGE(FAILED, "Destroy aicpu kernel failed."); + GELOGE(FAILED, "[Call][KernelLaunchEx] fail, model_id:%u, sub_model_id:%u, session_id:%lu", + model_id, sub_model_id, session_id); return FAILED; } } @@ -304,7 +297,7 @@ ge::Status ModelManager::DoLoadHybridModelOnline(uint32_t model_id, const string hybrid_model->SetModelId(model_id); hybrid_model->SetDeviceId(GetContext().DeviceId()); hybrid_model->SetOmName(om_name); - GE_CHK_STATUS_RET(hybrid_model->Init(), "Failed to init hybrid model. model_id = %u", model_id); + GE_CHK_STATUS_RET(hybrid_model->Init(), "[Init][HybridModel] failed. model_id = %u", model_id); auto shared_model = std::shared_ptr(hybrid_model.release()); InsertModel(model_id, shared_model); return SUCCESS; @@ -315,7 +308,8 @@ bool ModelManager::IsNeedHybridLoad(ge::GeRootModel &ge_root_model) { if (root_graph == nullptr) { REPORT_INNER_ERROR("E19999", "root graph in param ge_root_model is nullptr, model_id:%u, " "check invalid", ge_root_model.GetModelId()); - GELOGE(FAILED, "no model on root model"); + GELOGE(FAILED, "[Check][Param] root graph in param ge_root_model is nullptr, model_id:%u", + ge_root_model.GetModelId()); return false; } bool is_shape_unknown = root_graph->GetGraphUnknownFlag(); @@ -324,57 +318,6 @@ bool ModelManager::IsNeedHybridLoad(ge::GeRootModel &ge_root_model) { return is_shape_unknown || is_dsp_partitioned_graph || GetContext().GetHostExecFlag(); } -bool ModelManager::IsDumpSeverInited(uint64_t session_id) { - auto it = session_id_to_dump_server_init_flag_.find(session_id); - return it != session_id_to_dump_server_init_flag_.end() && it->second; -} - -Status ModelManager::AddDumpProperties(uint64_t session_id, const DumpProperties &dump_properties) { - if (!IsDumpSeverInited(session_id)) { - if (dump_properties.IsDumpOpen() || dump_properties.IsOpDebugOpen()) { - GE_IF_BOOL_EXEC(AdxDataDumpServerInit() != kDumpStatus, - GELOGE(PARAM_INVALID, "[Init][AdxDataDumpServer] failed, session_id:%lu.", session_id); - return PARAM_INVALID) - GELOGI("Init adx data dump server success"); - session_id_to_dump_server_init_flag_[session_id] = true; - } - } - DumpManager::GetInstance().AddDumpProperties(session_id, dump_properties); - return SUCCESS; -} - -Status ModelManager::InitDumPropertiesWithNewSessionId(uint64_t session_id) { - DumpProperties dump_properties; - dump_properties.InitByOptions(); - GE_CHK_STATUS_RET(AddDumpProperties(session_id, dump_properties), "[Add][DumpProperties] failed."); - return SUCCESS; -} - -Status ModelManager::UpdateSessionId(uint32_t model_id, GeModelPtr ge_model, - std::shared_ptr &davinci_model, uint64_t &session_id) { - uint64_t new_session_id; - Status ret = GenSessionId(new_session_id); - GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, return ret, "Generate session_id for infer failed."); - ret = davinci_model->UpdateSessionId(new_session_id); - GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, return ret, "Update session_id for infer failed."); - ge_model->InsertSessionMap(model_id, new_session_id); - GELOGD("Update new session id: %lu.", new_session_id); - session_id = new_session_id; - return SUCCESS; -} - -bool ModelManager::HasVarNode(ComputeGraphPtr &compute_graph) const { - for (ge::NodePtr &node : compute_graph->GetAllNodes()) { - if (node == nullptr) { - continue; - } - if (node->GetType() == VARIABLE) { - return true; - } - } - return false; -} - /// /// @ingroup domi_ome /// @brief load model online @@ -382,7 +325,7 @@ bool ModelManager::HasVarNode(ComputeGraphPtr &compute_graph) const { /// Status ModelManager::LoadModelOnline(uint32_t &model_id, const shared_ptr &ge_root_model, std::shared_ptr listener) { - GE_CHK_BOOL_RET_STATUS(listener.get() != nullptr, PARAM_INVALID, "Param incorrect, listener is null"); + GE_CHK_BOOL_RET_STATUS(listener.get() != nullptr, PARAM_INVALID, "[Check][Param] Param incorrect, listener is null"); if (model_id == INVALID_MODEL_ID) { GenModelId(&model_id); GELOGD("Generate new model_id:%u", model_id); @@ -411,21 +354,7 @@ Status ModelManager::LoadModelOnline(uint32_t &model_id, const shared_ptrAssign(ge_model)), GELOGW("assign model to modeldef failed."); break;); GE_TIMESTAMP_END(Assign, "GraphLoader::ModelAssign"); - /// In multi-threaded inference, using the same session_id among multiple threads may cause some threads to fail. - /// These session_ids come from the same model, so the values of session_id are the same. - /// Update session_id for infer in load model to avoid the same session_id. uint64_t session_id = GetContext().SessionId(); - // Inference graph with variable node is not support for multi-threads scenario - if (!ge_root_model->GetTrainFlag() && !HasVarNode(root_graph)) { - GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(UpdateSessionId(model_id, ge_model, davinci_model, session_id) != SUCCESS, - return ret, - "UpdateSessionId failed."); - GE_CHK_RT_RET(rtSetDevice(GetContext().DeviceId())); - GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(InitDumPropertiesWithNewSessionId(session_id) != SUCCESS, - GE_CHK_RT(rtDeviceReset(static_cast(GetContext().DeviceId()))); - return ret, - "Init DumProperties with new session_id failed."); - } const DumpProperties &dump_properties = DumpManager::GetInstance().GetDumpProperties(session_id); davinci_model->SetDumpProperties(dump_properties); @@ -444,13 +373,13 @@ Status ModelManager::LoadModelOnline(uint32_t &model_id, const shared_ptr &davinci_model) { - GE_CHK_BOOL_EXEC(davinci_model != nullptr, return, "davinci_model ptr is null, id: %u", model_id); + GE_CHK_BOOL_EXEC(davinci_model != nullptr, return, "[Check][Param] davinci_model ptr is null, id:%u", model_id); std::lock_guard lock(map_mutex_); model_map_[model_id] = davinci_model; } void ModelManager::InsertModel(uint32_t model_id, shared_ptr &hybrid_model) { - GE_CHK_BOOL_EXEC(hybrid_model != nullptr, return, "hybrid_model ptr is null, id: %u", model_id); + GE_CHK_BOOL_EXEC(hybrid_model != nullptr, return, "[Check][Param] hybrid_model ptr is null, id:%u", model_id); std::lock_guard lock(map_mutex_); hybrid_model_map_[model_id] = hybrid_model; } @@ -502,7 +431,7 @@ std::shared_ptr ModelManager::GetHybridModel(uint32_ } Status ModelManager::Unload(uint32_t model_id) { - GE_CHK_STATUS_RET(DeleteModel(model_id), "failed to unload model id: %u", model_id); + GE_CHK_STATUS_RET(DeleteModel(model_id), "[Delete][Model] failed, model id:%u", model_id); if (device_count > 0) { device_count--; GELOGI("Unload model %u success.", model_id); @@ -515,7 +444,7 @@ Status ModelManager::Unload(uint32_t model_id) { } Status ModelManager::UnloadModeldef(uint32_t model_id) { - GE_CHK_STATUS_RET(DeleteModel(model_id), "failed to unload modeldef id: %u", model_id); + GE_CHK_STATUS_RET(DeleteModel(model_id), "[Delete][Model] failed, model id: %u", model_id); return SUCCESS; } @@ -526,8 +455,8 @@ Status ModelManager::DataInput(const InputData &input_data, OutputData &output_d Status status = data_wrap->Init(input_data, output_data); if (status != SUCCESS) { - REPORT_CALL_ERROR("E19999", "Init InputDataWrapper failed, input data index: %u", input_data.index); - GELOGE(domi::PUSH_DATA_FAILED, "Init InputDataWrapper failed, input data index: %u.", input_data.index); + REPORT_CALL_ERROR("E19999", "Init InputDataWrapper failed, input data index:%u", input_data.index); + GELOGE(domi::PUSH_DATA_FAILED, "[Init][InputDataWrapper] failed, input data index:%u.", input_data.index); return domi::PUSH_DATA_FAILED; } @@ -536,7 +465,8 @@ Status ModelManager::DataInput(const InputData &input_data, OutputData &output_d std::shared_ptr model = GetModel(model_id); - GE_CHK_BOOL_RET_STATUS(model != nullptr, PARAM_INVALID, "Invalid model id %u in InputData! ", model_id); + GE_CHK_BOOL_RET_STATUS(model != nullptr, PARAM_INVALID, + "[Get][Model] failed, Invalid model id %u in InputData!", model_id); GE_IF_BOOL_EXEC(model->GetDataInputTid() == 0, model->SetDataInputTid(mmGetTid())); @@ -544,7 +474,7 @@ Status ModelManager::DataInput(const InputData &input_data, OutputData &output_d GE_CHECK_NOTNULL(inputer); if (inputer->Push(data_wrap) != SUCCESS) { REPORT_CALL_ERROR("E19999", "DataInputer queue is full, please call again later, model_id %u", model_id); - GELOGE(domi::DATA_QUEUE_ISFULL, "Data queue is full, please call again later, model_id %u ", model_id); + GELOGE(domi::DATA_QUEUE_ISFULL, "[Call][Push] Data queue is full, please call again later, model_id %u ", model_id); return domi::DATA_QUEUE_ISFULL; } GELOGD("Data input success, model id:%u", model_id); @@ -558,10 +488,9 @@ Status ModelManager::GetCurDynamicDims(const vector> &user_real_ GELOGD("Start get cur dynamic dims."); if (user_real_input_dims.size() != user_input_dims.size()) { REPORT_INNER_ERROR("E19999", "Param user_real_input_dims.size:%zu != user_input_dims.size:%zu, " - "check invalid", - user_real_input_dims.size(), user_input_dims.size()); + "check invalid", user_real_input_dims.size(), user_input_dims.size()); GELOGE(INTERNAL_ERROR, - "The input count of user: %zu should be equal to the data count of graph: %zu", + "[Check][Param] The input count of user:%zu should be equal to the data count of graph:%zu", user_real_input_dims.size(), user_input_dims.size()); return INTERNAL_ERROR; } @@ -571,8 +500,8 @@ Status ModelManager::GetCurDynamicDims(const vector> &user_real_ REPORT_INNER_ERROR("E19999", "Param user_real_input_dims[%zu].size:%zu != user_input_dims[%zu].size:%zu, " "check invalid", i, user_real_input_dims[i].size(), i, user_input_dims[i].second.size()); - GELOGE(INTERNAL_ERROR, - "The shape size: %zu of dynamic input: %s should be equal to the shape size of input shape: %zu.", + GELOGE(INTERNAL_ERROR, "[Check][Param] The shape size:%zu of dynamic input:%s " + "should be equal to the shape size of input shape:%zu.", user_real_input_dims[i].size(), user_input_dims[i].first.c_str(), user_input_dims[i].second.size()); return INTERNAL_ERROR; } @@ -594,7 +523,7 @@ Status ModelManager::GetCurDynamicDims(const vector> &user_real_ if (!cur_dynamic_dims_valid) { REPORT_INNER_ERROR("E19999", "cur dynamic dims is %s, not exist in options, check invalid", formats::JoinToString(cur_dynamic_dims).c_str()); - GELOGE(INTERNAL_ERROR, "Cur dynamic dims is %s, not exist in options.", + GELOGE(INTERNAL_ERROR, "[Check][Param] Cur dynamic dims is %s, not exist in options.", formats::JoinToString(cur_dynamic_dims).c_str()); return INTERNAL_ERROR; } @@ -632,15 +561,16 @@ Status ModelManager::DataInputTensor(uint32_t model_id, const std::vector(cur_dynamic_dims.size() * sizeof(int32_t)); - GE_CHK_BOOL_EXEC(memcpy_s(data.data, length, cur_dynamic_dims.data(), length) == EOK, return INTERNAL_ERROR, - "Failed to memcpy data."); + GE_CHK_BOOL_EXEC(memcpy_s(data.data, length, cur_dynamic_dims.data(), length) == EOK, + REPORT_CALL_ERROR("E19999", "memcpy data failed, size:%u", length); + return INTERNAL_ERROR, "[Memcpy][Data] failed, size:%u.", length); data.length = length; input_data.blobs.push_back(data); } @@ -654,21 +584,22 @@ Status ModelManager::DataInputTensor(uint32_t model_id, const std::vectorInit(input_data, output_data), return domi::PUSH_DATA_FAILED, - "Init InputDataWrapper failed,input data model_id is : %u.", model_id); + "[Init][InputDataWrapper] failed, input data model_id:%u.", model_id); if (hybrid_model != nullptr) { - GE_CHK_STATUS_RET(hybrid_model->EnqueueData(data_wrap), "Data queue is full, please call again later, model_id %u ", - model_id); + GE_CHK_STATUS_RET(hybrid_model->EnqueueData(data_wrap), + "[Enqueue][Data] Data queue is full, please call again later, model_id:%u", model_id); return SUCCESS; } - GE_CHK_BOOL_RET_STATUS(model != nullptr, PARAM_INVALID, "Invalid model id %u in InputData! ", model_id); + GE_CHK_BOOL_RET_STATUS(model != nullptr, PARAM_INVALID, + "[Check][Param] Invalid model id %u in InputData!", model_id); DataInputer *inputer = model->GetDataInputer(); GE_CHECK_NOTNULL(inputer); GE_CHK_STATUS_EXEC(inputer->Push(data_wrap), return domi::DATA_QUEUE_ISFULL, - "Data queue is full, please call again later, model_id %u ", model_id); + "[Call][Push] Data queue is full, please call again later, model_id %u ", model_id); GELOGD("Data input success, model id:%u", model_id); @@ -691,7 +622,8 @@ Status ModelManager::Start(uint32_t model_id) { std::shared_ptr davinci_model = GetModel(model_id); - GE_CHK_BOOL_RET_STATUS(davinci_model != nullptr, PARAM_INVALID, "Invalid model id %u to start! ", model_id); + GE_CHK_BOOL_RET_STATUS(davinci_model != nullptr, PARAM_INVALID, + "[Get][Model] failed, Invalid model id %u to start! ", model_id); Status status = davinci_model->ModelRunStart(); if (status == SUCCESS) { @@ -718,7 +650,8 @@ Status ModelManager::Stop(uint32_t model_id) { } std::shared_ptr davinci_model = GetModel(model_id); - GE_CHK_BOOL_RET_STATUS(davinci_model != nullptr, PARAM_INVALID, "Invalid model id %u to stop!", model_id); + GE_CHK_BOOL_RET_STATUS(davinci_model != nullptr, PARAM_INVALID, + "[Get][Model] failed, Invalid model id %u to stop!", model_id); Status status = davinci_model->ModelRunStop(); if (status == SUCCESS) { @@ -746,9 +679,8 @@ Status ModelManager::HandleCommand(const Command &command) { auto iter = cmds.find(command.cmd_type); if (iter == cmds.end()) { - REPORT_INNER_ERROR("E19999", "Unsupported command:%s check", - command.cmd_type.c_str()); - GELOGE(PARAM_INVALID, "Unsupported command: %s", command.cmd_type.c_str()); + REPORT_INNER_ERROR("E19999", "Unsupported command:%s check", command.cmd_type.c_str()); + GELOGE(PARAM_INVALID, "[Check][Param] Unsupported command:%s", command.cmd_type.c_str()); return PARAM_INVALID; } else { return iter->second(command); @@ -761,8 +693,8 @@ Status ModelManager::GetModelByCmd(const Command &command, REPORT_INNER_ERROR("E19999", "command.cmd_params.size:%zu < kCmdParSize:%u, command_type:%s, " "check invalid", command.cmd_params.size(), kCmdParSize, command.cmd_type.c_str()); - GELOGE(PARAM_INVALID, "When the cmd_type is '%s', the size of cmd_params must larger than 2.", - command.cmd_type.c_str()); + GELOGE(PARAM_INVALID, "[Check][Param] When the cmd_type is '%s', the size of cmd_params must larger than 2.", + command.cmd_type.c_str()); return PARAM_INVALID; } @@ -773,19 +705,16 @@ Status ModelManager::GetModelByCmd(const Command &command, try { model_id = std::stoi(value); } catch (std::invalid_argument &) { - REPORT_INNER_ERROR("E19999", "%s param:%s, check invalid", PROFILE_MODEL_ID.c_str(), - value.c_str()); - GELOGE(PARAM_INVALID, "Model id: %s is invalid.", value.c_str()); + REPORT_INNER_ERROR("E19999", "%s param:%s, check invalid", PROFILE_MODEL_ID.c_str(), value.c_str()); + GELOGE(PARAM_INVALID, "[Check][Param] Model id:%s is invalid.", value.c_str()); return PARAM_INVALID; } catch (std::out_of_range &) { - REPORT_INNER_ERROR("E19999", "%s param:%s, check out of range", PROFILE_MODEL_ID.c_str(), - value.c_str()); - GELOGE(PARAM_INVALID, "Model id: %s is out of range.", value.c_str()); + REPORT_INNER_ERROR("E19999", "%s param:%s, check out of range", PROFILE_MODEL_ID.c_str(), value.c_str()); + GELOGE(PARAM_INVALID, "[Check][Param] Model id:%s is out of range.", value.c_str()); return PARAM_INVALID; } catch (...) { - REPORT_INNER_ERROR("E19999", "%s param:%s, check cannot change to int", - PROFILE_MODEL_ID.c_str(), value.c_str()); - GELOGE(FAILED, "Model id: %s cannot change to int.", value.c_str()); + REPORT_INNER_ERROR("E19999", "%s param:%s, check cannot change to int", PROFILE_MODEL_ID.c_str(), value.c_str()); + GELOGE(FAILED, "[Check][Param] Model id:%s cannot change to int.", value.c_str()); return FAILED; } @@ -793,15 +722,13 @@ Status ModelManager::GetModelByCmd(const Command &command, GE_CHECK_NOTNULL(model_manager); davinci_model = model_manager->GetModel(static_cast(model_id)); if (davinci_model == nullptr) { - REPORT_INNER_ERROR("E19999", "GetModel from model_manager fail, model_id:%u", - model_id); - GELOGE(FAILED, "Model id: %d is invaild or model is not loaded.", model_id); + REPORT_INNER_ERROR("E19999", "GetModel from model_manager fail, model_id:%u", model_id); + GELOGE(FAILED, "[Get][Model] failed, Model id:%d is invaild or model is not loaded.", model_id); return FAILED; } } else { - REPORT_INNER_ERROR("E19999", "Fisrt cmd_param not %s, check invalid", - PROFILE_MODEL_ID.c_str()); - GELOGE(FAILED, "The model_id parameter is not found in the command."); + REPORT_INNER_ERROR("E19999", "Fisrt cmd_param not %s, check invalid", PROFILE_MODEL_ID.c_str()); + GELOGE(FAILED, "[Check][Param] The model_id parameter is not found in the command."); return FAILED; } @@ -817,7 +744,8 @@ Status ModelManager::HandleProfModelSubscribeCommand(const Command &command) { if (ProfilingManager::Instance().ProfModelSubscribe(command.module_index, static_cast(davinci_model.get())) != SUCCESS) { - GELOGE(FAILED, "Handle prof model subscribe failed."); + GELOGE(FAILED, "[Handle][ProfModelSubscribe] failed, module_index:%lu.", + command.module_index); return FAILED; } @@ -832,7 +760,7 @@ Status ModelManager::HandleProfModelUnsubscribeCommand(const Command &command) { } if (ProfilingManager::Instance().ProfModelUnsubscribe(static_cast(davinci_model.get())) != SUCCESS) { - GELOGE(FAILED, "Handle prof model unsubscribe failed."); + GELOGE(FAILED, "[Handle][ProfModelUnsubscribe] failed."); return FAILED; } @@ -842,7 +770,7 @@ Status ModelManager::HandleProfModelUnsubscribeCommand(const Command &command) { Status ModelManager::HandleProfInitCommand(const Command &command) { uint64_t module_index = command.module_index; if (ProfilingManager::Instance().ProfInit(module_index) != SUCCESS) { - GELOGE(FAILED, "Handle prof init failed."); + GELOGE(FAILED, "[Handle][ProfInit] failed, module_index:%lu.", module_index); return FAILED; } return SUCCESS; @@ -850,7 +778,7 @@ Status ModelManager::HandleProfInitCommand(const Command &command) { Status ModelManager::HandleProfFinalizeCommand(const Command &command) { if (ProfilingManager::Instance().ProfFinalize() != SUCCESS) { - GELOGE(FAILED, "Handle prof finalize failed."); + GELOGE(FAILED, "[Handle][ProfFinalize] failed."); return FAILED; } return SUCCESS; @@ -866,13 +794,14 @@ Status ModelManager::HandleProfStartCommand(const Command &command) { if (command.cmd_params.size() < kProfStartCmdParaSize) { REPORT_INNER_ERROR("E19999", "command.cmd_params.size:%zu < %zu, check invalid", command.cmd_params.size(), kProfStartCmdParaSize); - GELOGE(PARAM_INVALID, "When the cmd_type is 'profile start', the size of cmd_params must larger than 2."); + GELOGE(PARAM_INVALID, "[Check][Param] When the cmd_type is 'profile start', " + "the size:%zu of cmd_params must larger than 2.", command.cmd_params.size()); return PARAM_INVALID; } if (command.cmd_params.size() > kProfCmdParaMaxSize) { REPORT_INNER_ERROR("E19999", "command.cmd_params.size:%zu > %zu, check invalid", command.cmd_params.size(), kProfCmdParaMaxSize); - GELOGE(PARAM_INVALID, "Command para size[%zu] larger than max[1000].", command.cmd_params.size()); + GELOGE(PARAM_INVALID, "[Check][Param] Command param size[%zu] larger than max[1000].", command.cmd_params.size()); return PARAM_INVALID; } @@ -886,7 +815,7 @@ Status ModelManager::HandleProfStartCommand(const Command &command) { } uint64_t module_index = command.module_index; if (ProfilingManager::Instance().ProfStartProfiling(module_index, cmd_params_map) != SUCCESS) { - GELOGE(FAILED, "Handle prof start failed."); + GELOGE(FAILED, "[Handle][ProfStartProfiling] failed, module_index:%lu.", module_index); return FAILED; } return SUCCESS; @@ -896,13 +825,14 @@ Status ModelManager::HandleProfStopCommand(const Command &command) { if (command.cmd_params.size() < kProfStartCmdParaSize) { REPORT_INNER_ERROR("E19999", "command.cmd_params.size:%zu < %zu, check invalid", command.cmd_params.size(), kProfStartCmdParaSize); - GELOGE(PARAM_INVALID, "When the cmd_type is 'profile stop', the size of cmd_params must larger than 2."); + GELOGE(PARAM_INVALID, "[Check][Param] When the cmd_type is 'profile stop', " + "the size:%zu of cmd_params must larger than 2.", command.cmd_params.size()); return PARAM_INVALID; } if (command.cmd_params.size() > kProfCmdParaMaxSize) { REPORT_INNER_ERROR("E19999", "command.cmd_params.size:%zu > %zu, check invalid", command.cmd_params.size(), kProfCmdParaMaxSize); - GELOGE(PARAM_INVALID, "Command para size[%zu] larger than max[1000].", command.cmd_params.size()); + GELOGE(PARAM_INVALID, "[Check][Param] Command param size[%zu] larger than max[1000].", command.cmd_params.size()); return PARAM_INVALID; } @@ -916,7 +846,7 @@ Status ModelManager::HandleProfStopCommand(const Command &command) { } uint64_t module_index = command.module_index; if (ProfilingManager::Instance().ProfStopProfiling(module_index, cmd_params_map) != SUCCESS) { - GELOGE(FAILED, "Handle prof finalize failed."); + GELOGE(FAILED, "[Handle][ProfStopProfiling] failed, module_index:%lu.", module_index); return FAILED; } return SUCCESS; @@ -927,9 +857,8 @@ static Status ParserPara(const Command &command, const string &dump_key, string if (iter != command.cmd_params.end()) { ++iter; if (iter == command.cmd_params.end()) { - REPORT_INNER_ERROR("E19999", "dump_key:%s can't find in command.param, check invalid", - dump_key.c_str()); - GELOGE(PARAM_INVALID, "Invalid access."); + REPORT_INNER_ERROR("E19999", "dump_key:%s can't find in command.param, check invalid", dump_key.c_str()); + GELOGE(PARAM_INVALID, "[Check][Param] dump_key:%s can't find in command.param, check invalid", dump_key.c_str()); return PARAM_INVALID; } dump_value = *iter; @@ -939,9 +868,9 @@ static Status ParserPara(const Command &command, const string &dump_key, string Status ModelManager::HandleDumpCommand(const Command &command) { if (command.cmd_params.size() % kDumpCmdPairSize != 0) { - REPORT_INNER_ERROR("E19999", "command.cmd_params.size:%zu MOD 2 != 0, check invalid", - command.cmd_params.size()); - GELOGE(PARAM_INVALID, "When the cmd_type is 'dump', the size of cmd_params must be a even number."); + REPORT_INNER_ERROR("E19999", "command.cmd_params.size:%zu MOD 2 != 0, check invalid", command.cmd_params.size()); + GELOGE(PARAM_INVALID, "[Check][Param] When the cmd_type is 'dump', " + "the size:%zu of cmd_params must be a even number.", command.cmd_params.size()); return PARAM_INVALID; } @@ -953,14 +882,14 @@ Status ModelManager::HandleDumpCommand(const Command &command) { auto ret = ParserPara(command, DUMP_STATUS, dump_status); if (ret != SUCCESS) { - GELOGE(PARAM_INVALID, "parser dump status failed"); + GELOGE(PARAM_INVALID, "[Parser][DumpStatus] failed, ret:%d", ret); return FAILED; } GELOGI("dump status = %s.", dump_status.c_str()); ret = ParserPara(command, DUMP_MODEL, dump_model); if (ret != SUCCESS) { - GELOGE(PARAM_INVALID, "parser dump model failed"); + GELOGE(PARAM_INVALID, "[Parser][DumpModel] failed, ret:%d", ret); return FAILED; } GELOGI("dump model = %s.", dump_model.c_str()); @@ -979,7 +908,7 @@ Status ModelManager::HandleDumpCommand(const Command &command) { ret = ParserPara(command, DUMP_FILE_PATH, dump_path); if (ret != SUCCESS) { - GELOGE(PARAM_INVALID, "parser dump path failed"); + GELOGE(PARAM_INVALID, "[Parser][DumpPath] failed, ret:%d", ret); return FAILED; } if (!dump_path.empty() && dump_path[dump_path.size() - 1] != '/') { @@ -990,7 +919,7 @@ Status ModelManager::HandleDumpCommand(const Command &command) { ret = ParserPara(command, DUMP_MODE, dump_mode); if (ret != SUCCESS) { - GELOGE(PARAM_INVALID, "parser dump mode failed"); + GELOGE(PARAM_INVALID, "[Parser][DumpMode] failed, ret:%d", ret); return FAILED; } GELOGI("dump mode = %s", dump_mode.c_str()); @@ -1010,8 +939,8 @@ Status ModelManager::GetMaxUsedMemory(const uint32_t model_id, uint64_t &max_siz } std::shared_ptr davinci_model = GetModel(model_id); - GE_CHK_BOOL_RET_STATUS(davinci_model != nullptr, PARAM_INVALID, "GetMaxUsedMemory Failed, Invalid model id %u!", - model_id); + GE_CHK_BOOL_RET_STATUS(davinci_model != nullptr, PARAM_INVALID, + "[Get][Model] failed, Invalid model id:%u!", model_id); max_size = davinci_model->TotalMemSize(); return SUCCESS; @@ -1020,8 +949,8 @@ Status ModelManager::GetMaxUsedMemory(const uint32_t model_id, uint64_t &max_siz Status ModelManager::GetInputOutputDescInfo(const uint32_t model_id, vector &input_desc, vector &output_desc) { std::shared_ptr davinci_model = GetModel(model_id); - GE_CHK_BOOL_RET_STATUS(davinci_model != nullptr, PARAM_INVALID, "GetInputOutputDescInfo Failed, Invalid model id %u!", - model_id); + GE_CHK_BOOL_RET_STATUS(davinci_model != nullptr, PARAM_INVALID, + "[Get][Model] failed, Invalid model id %u!", model_id); return davinci_model->GetInputOutputDescInfo(input_desc, output_desc); } @@ -1038,7 +967,7 @@ Status ModelManager::GetInputOutputDescInfo(const uint32_t model_id, vector davinci_model = GetModel(model_id); GE_CHK_BOOL_RET_STATUS(davinci_model != nullptr, ACL_ERROR_GE_EXEC_MODEL_ID_INVALID, - "GetInputOutputDescInfo Failed, Invalid model id %u!", model_id); + "[Get][Model] Failed, Invalid model id %u!", model_id); return davinci_model->GetInputOutputDescInfo(input_desc, output_desc, inputFormats, outputFormats, new_model_desc); } @@ -1059,7 +988,7 @@ Status ModelManager::GetDynamicBatchInfo(const uint32_t model_id, std::vector davinci_model = GetModel(model_id); GE_CHK_BOOL_RET_STATUS(davinci_model != nullptr, ACL_ERROR_GE_EXEC_MODEL_ID_INVALID, - "GetDynamicBatchInfo failed, Invalid model id %u!", model_id); + "[Get][Model] failed, Invalid model id %u!", model_id); return davinci_model->GetDynamicBatchInfo(batch_info, dynamic_type); } @@ -1074,7 +1003,7 @@ Status ModelManager::GetDynamicBatchInfo(const uint32_t model_id, std::vector> &batch_info) { std::shared_ptr davinci_model = GetModel(model_id); GE_CHK_BOOL_RET_STATUS(davinci_model != nullptr, ACL_ERROR_GE_EXEC_MODEL_ID_INVALID, - "GetCombinedDynamicDims Failed, Invalid Model ID %u!", model_id); + "[Get][Model] Failed, Invalid Model ID %u!", model_id); davinci_model->GetCombinedDynamicDims(batch_info); return SUCCESS; @@ -1097,7 +1026,7 @@ Status ModelManager::GetUserDesignateShapeOrder(const uint32_t model_id, auto davinci_model = GetModel(model_id); GE_CHK_BOOL_RET_STATUS(davinci_model != nullptr, ACL_ERROR_GE_EXEC_MODEL_ID_INVALID, - "GetUserDesignateShapeOrder Failed, Invalid Model ID %u!", model_id) + "[Get][Model] Failed, Invalid Model ID %u!", model_id) davinci_model->GetUserDesignateShapeOrder(user_input_shape_order); return SUCCESS; } @@ -1105,7 +1034,7 @@ Status ModelManager::GetUserDesignateShapeOrder(const uint32_t model_id, Status ModelManager::GetCurShape(const uint32_t model_id, std::vector &batch_info, int32_t &dynamic_type) { auto davinci_model = GetModel(model_id); GE_CHK_BOOL_RET_STATUS(davinci_model != nullptr, ACL_ERROR_GE_EXEC_MODEL_ID_INVALID, - "GetCurShape Failed, Invalid Model ID %u!", model_id); + "[Get][Model] Failed, Invalid Model ID %u!", model_id); davinci_model->GetCurShape(batch_info, dynamic_type); return SUCCESS; } @@ -1134,7 +1063,7 @@ Status ModelManager::GetModelAttr(uint32_t model_id, std::vector &dynami std::shared_ptr davinci_model = GetModel(model_id); GE_CHK_BOOL_RET_STATUS(davinci_model != nullptr, ACL_ERROR_GE_EXEC_MODEL_ID_INVALID, - "GetModelAttr Failed, Invalid Model ID %u!", model_id); + "[Get][Model] Failed, Invalid Model ID %u!", model_id); davinci_model->GetModelAttr(dynamic_output_shape_info); return SUCCESS; } @@ -1150,14 +1079,14 @@ Status ModelManager::GetModelAttr(uint32_t model_id, std::vector &dynami Status ModelManager::GetAippInfo(const uint32_t model_id, uint32_t index, AippConfigInfo &aipp_info) { std::shared_ptr davinci_model = GetModel(model_id); GE_CHK_BOOL_RET_STATUS(davinci_model != nullptr, ACL_ERROR_GE_EXEC_MODEL_ID_INVALID, - "GetAIPPInfo failed, invalid model_id is %u.", model_id); + "[Get][Model] failed, invalid model_id is %u.", model_id); return davinci_model->GetAippInfo(index, aipp_info); } Status ModelManager::GetAippType(uint32_t model_id, uint32_t index, InputAippType &type, size_t &aipp_index) { std::shared_ptr davinci_model = GetModel(model_id); GE_CHK_BOOL_RET_STATUS(davinci_model != nullptr, ACL_ERROR_GE_EXEC_MODEL_ID_INVALID, - "GetAIPPInfo failed, invalid model_id is %u.", model_id); + "[Get][Model] failed, invalid model_id is %u.", model_id); return davinci_model->GetAippType(index, type, aipp_index); } @@ -1173,7 +1102,7 @@ Status ModelManager::GenSessionId(uint64_t &session_id) { mmTimeval tv; if (mmGetTimeOfDay(&tv, nullptr) != 0) { REPORT_CALL_ERROR("E19999", "Call mmGetTimeOfDay fail. errmsg:%s", strerror(errno)); - GELOGE(INTERNAL_ERROR, "Failed to get current time."); + GELOGE(INTERNAL_ERROR, "[Call][MmGetTimeOfDay] fail. errmsg:%s", strerror(errno)); return INTERNAL_ERROR; } uint64_t timestamp = static_cast(tv.tv_sec * kTimeSpecMiro + tv.tv_usec); // 1000000us @@ -1192,7 +1121,8 @@ Status ModelManager::GenSessionId(uint64_t &session_id) { Status ModelManager::LoadModelOffline(uint32_t &model_id, const ModelData &model, shared_ptr listener, void *dev_ptr, size_t mem_size, void *weight_ptr, size_t weight_size) { GE_CHK_BOOL_RET_STATUS(model.key.empty() || mmAccess2(model.key.c_str(), M_F_OK) == EN_OK, - ACL_ERROR_GE_PARAM_INVALID, "Input key file path %s is invalid, %s", model.key.c_str(), strerror(errno)); + ACL_ERROR_GE_PARAM_INVALID, + "[Check][Param] Input key file path %s is invalid, %s", model.key.c_str(), strerror(errno)); GenModelId(&model_id); mmTimespec timespec = mmGetTickCount(); @@ -1200,14 +1130,14 @@ Status ModelManager::LoadModelOffline(uint32_t &model_id, const ModelData &model ModelHelper model_helper; Status ret = model_helper.LoadRootModel(model); if (ret != SUCCESS) { - GELOGE(ret, "load model failed."); + GELOGE(ret, "[Load][RootModel] failed, ret:%d, model_id:%u.", ret, model_id); return ret; } if (model_helper.GetModelType()) { bool is_shape_unknown = false; GE_CHK_STATUS_RET(model_helper.GetGeRootModel()->CheckIsUnknownShape(is_shape_unknown), - "CheckIsUnknownShape failed, model id:%u", model_id); + "[Check][IsUnknownShape] failed, model id:%u", model_id); if (is_shape_unknown || GetContext().GetHostExecFlag()) { return DoLoadHybridModelOnline(model_id, model.om_name, model_helper.GetGeRootModel(), listener); } @@ -1218,7 +1148,7 @@ Status ModelManager::LoadModelOffline(uint32_t &model_id, const ModelData &model shared_ptr davinci_model = MakeShared(model.priority, listener); if (davinci_model == nullptr) { REPORT_CALL_ERROR("E19999", "New DavinciModel fail"); - GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "Make shared failed"); + GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "[New][DavinciModel] fail"); return ACL_ERROR_GE_MEMORY_ALLOCATION; } davinci_model->SetProfileTime(MODEL_LOAD_START, (timespec.tv_sec * kTimeSpecNano + @@ -1234,7 +1164,7 @@ Status ModelManager::LoadModelOffline(uint32_t &model_id, const ModelData &model rtError_t rt_ret = rtGetDevice(&device_id); if (rt_ret != RT_ERROR_NONE || device_id < 0) { REPORT_CALL_ERROR("E19999", "Call rtGetDevice failed, ret = 0x%X", rt_ret); - GELOGE(rt_ret, "Call rtGetDevice failed, ret = 0x%X, device_id = %d.", rt_ret, device_id); + GELOGE(rt_ret, "[Call][RtGetDevice] failed, ret = 0x%X, device_id = %d.", rt_ret, device_id); return RT_ERROR_TO_GE_STATUS(rt_ret); } davinci_model->SetDeviceId(device_id); @@ -1250,12 +1180,13 @@ Status ModelManager::LoadModelOffline(uint32_t &model_id, const ModelData &model /// Update session_id for infer in load model to avoid the same session_id. uint64_t new_session_id; ret = GenSessionId(new_session_id); - GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, break, "Generate session_id for inference failed."); + GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, break, "[Generate][SessionId] for inference failed, ret:%d.", ret); ret = davinci_model->UpdateSessionId(new_session_id); - GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, break, "Update session_id for inference failed."); + GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, break, + "[Update][SessionId] for inference failed, session id:%lu.", new_session_id); ret = davinci_model->Init(dev_ptr, mem_size, weight_ptr, weight_size); - GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, break, "DavinciInit failed."); + GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, break, "[Init][DavinciModel] failed, ret:%d.", ret); InsertModel(model_id, davinci_model); @@ -1280,26 +1211,27 @@ Status ModelManager::LoadModelWithQ(uint32_t &model_id, const ModelData &model_d const std::vector &input_queue_ids, const std::vector &output_queue_ids) { GE_CHK_BOOL_RET_STATUS(model_data.key.empty() || mmAccess2(model_data.key.c_str(), M_F_OK) == EN_OK, - ACL_ERROR_GE_PARAM_INVALID, "input key file path %s is not valid, %s", + ACL_ERROR_GE_PARAM_INVALID, + "[Check][Param] input key file path %s is not valid, %s", model_data.key.c_str(), strerror(errno)); ModelHelper model_helper; Status ret = model_helper.LoadModel(model_data); if (ret != SUCCESS) { - GELOGE(ret, "load model failed."); + GELOGE(ret, "[Load][Model] failed."); return ret; } shared_ptr davinci_model = MakeShared(model_data.priority, nullptr); if (davinci_model == nullptr) { REPORT_CALL_ERROR("E19999", "New DavinciModel fail"); - GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "create model failed."); + GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "[Create][Model] failed."); return ACL_ERROR_GE_MEMORY_ALLOCATION; } ret = davinci_model->Assign(model_helper.GetGeModel()); if (ret != SUCCESS) { - GELOGE(ret, "assign model failed."); + GELOGE(ret, "[Assign][Model] failed, ret:%d.", ret); return ret; } @@ -1308,15 +1240,17 @@ Status ModelManager::LoadModelWithQ(uint32_t &model_id, const ModelData &model_d /// Update session_id for infer in load model to avoid the same session_id. uint64_t new_session_id; ret = GenSessionId(new_session_id); - GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, return ret, "Generate session_id for infer failed."); + GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, return ret, + "[Generate][SessionId] for infer failed, ret:%d.", ret); ret = davinci_model->UpdateSessionId(new_session_id); - GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, return ret, "Update session_id for infer failed."); + GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, return ret, + "[Update][SessionId] for infer failed, SessionId:%lu.", new_session_id); GenModelId(&model_id); davinci_model->SetId(model_id); ret = davinci_model->SetQueIds(input_queue_ids, output_queue_ids); if (ret != SUCCESS) { - GELOGE(ret, "set model queue ids failed."); + GELOGE(ret, "[Set][Ids] for model queue failed, ret:%d, model_id:%u.", ret, model_id); return ret; } @@ -1324,7 +1258,7 @@ Status ModelManager::LoadModelWithQ(uint32_t &model_id, const ModelData &model_d ret = davinci_model->Init(); if (ret != SUCCESS) { - GELOGE(ret, "init model failed."); + GELOGE(ret, "[Init][Model] failed, ret:%d, model_id:%u.", ret, model_id); return ret; } @@ -1362,7 +1296,7 @@ Status ModelManager::ExecuteModel(uint32_t model_id, rtStream_t stream, bool asy std::shared_ptr davinci_model = GetModel(model_id); GE_CHK_BOOL_RET_STATUS(davinci_model != nullptr, ACL_ERROR_GE_EXEC_MODEL_ID_INVALID, - "Invalid model id %u, check whether model has been loaded or not.", model_id); + "[Get][Model] Invalid model id %u, check whether model has been loaded or not.", model_id); if (davinci_model->NeedDestroyAicpuKernel()) { GELOGI("Start to destroy specified aicpu kernel."); @@ -1413,9 +1347,8 @@ Status ModelManager::LoadCustAicpuSo(const OpDescPtr &op_desc, const string &so_ rtContext_t rt_cur_ctx = nullptr; auto rt_error = rtCtxGetCurrent(&rt_cur_ctx); if (rt_error != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtCtxGetCurrent failed, ret = 0x%X", - rt_error); - GELOGE(RT_FAILED, "get current context failed, runtime result is %d", static_cast(rt_error)); + REPORT_CALL_ERROR("E19999", "Call rtCtxGetCurrent failed, ret = 0x%X", rt_error); + GELOGE(RT_FAILED, "[Call][RtCtxGetCurrent] failed, runtime result is %d", static_cast(rt_error)); return RT_FAILED; } @@ -1450,9 +1383,8 @@ Status ModelManager::LaunchKernelCustAicpuSo(const string &kernel_name) { rtContext_t rt_cur_ctx = nullptr; auto rt_error = rtCtxGetCurrent(&rt_cur_ctx); if (rt_error != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtCtxGetCurrent failed, ret = 0x%X", - rt_error); - GELOGE(RT_FAILED, "get current context failed, runtime result is %d", static_cast(rt_error)); + REPORT_CALL_ERROR("E19999", "Call rtCtxGetCurrent failed, ret = 0x%X", rt_error); + GELOGE(RT_FAILED, "[Call][RtCtxGetCurrent] failed, runtime result is %d", static_cast(rt_error)); return RT_FAILED; } uintptr_t resource_id = reinterpret_cast(rt_cur_ctx); @@ -1477,17 +1409,15 @@ Status ModelManager::LaunchKernelCustAicpuSo(const string &kernel_name) { status = rtMalloc(&d_aicpu_data, aicpu_data_length, RT_MEMORY_HBM); if (status != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%u, ret = 0x%X", - aicpu_data_length, status); - GELOGE(RT_FAILED, "Call rt failed, status: 0x%x", status); + REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%u, ret = 0x%X", aicpu_data_length, status); + GELOGE(RT_FAILED, "[Call][RtMalloc] failed, size:%u, ret = 0x%X", aicpu_data_length, status); return RT_ERROR_TO_GE_STATUS(status); } allocated_mem.push_back(d_aicpu_data); status = rtMalloc(&d_so_name, so_name.size(), RT_MEMORY_HBM); if (status != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%zu, ret = 0x%X", - so_name.size(), status); - GELOGE(RT_FAILED, "Call rt failed, status: 0x%x", status); + REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%zu, ret = 0x%X", so_name.size(), status); + GELOGE(RT_FAILED, "[Call][RtMalloc] fail, size:%zu, ret = 0x%X", so_name.size(), status); return RT_ERROR_TO_GE_STATUS(status); } allocated_mem.push_back(d_so_name); @@ -1509,9 +1439,8 @@ Status ModelManager::LaunchKernelCustAicpuSo(const string &kernel_name) { uint32_t args_size = sizeof(CustAicpuSoBuf) * v_cust_so.size(); status = rtMalloc(&args, args_size, RT_MEMORY_HBM); if (status != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%u, ret = 0x%X", - args_size, status); - GELOGE(RT_FAILED, "Call rt failed, status: 0x%x", status); + REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%u, ret = 0x%X", args_size, status); + GELOGE(RT_FAILED, "[Call][RtMalloc] fail, size:%u, ret = 0x%X", args_size, status); return RT_ERROR_TO_GE_STATUS(status); } allocated_mem.push_back(args); @@ -1525,9 +1454,8 @@ Status ModelManager::LaunchKernelCustAicpuSo(const string &kernel_name) { uint32_t batch_args_size = sizeof(BatchLoadOpFromBufArgs); status = rtMalloc(&batch_args, batch_args_size, RT_MEMORY_HBM); if (status != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%u, ret = 0x%X", - batch_args_size, status); - GELOGE(RT_FAILED, "Call rt failed, status: 0x%x", status); + REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%u, ret = 0x%X", batch_args_size, status); + GELOGE(RT_FAILED, "[Call][RtMalloc] fail, size:%u, ret = 0x%X", batch_args_size, status); return RT_ERROR_TO_GE_STATUS(status); } allocated_mem.push_back(batch_args); @@ -1539,9 +1467,8 @@ Status ModelManager::LaunchKernelCustAicpuSo(const string &kernel_name) { status = rtStreamSynchronize(stream); if (status != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtStreamSynchronize fail, ret = 0x%X", - status); - GELOGE(RT_FAILED, "Call rt stream sync failed, status: 0x%x", status); + REPORT_CALL_ERROR("E19999", "Call rtStreamSynchronize fail, ret = 0x%X", status); + GELOGE(RT_FAILED, "[Call][RtStreamSynchronize] fail, ret = 0x%X", status); return RT_ERROR_TO_GE_STATUS(status); } std::function callback = [&]() { @@ -1556,12 +1483,14 @@ Status ModelManager::LaunchKernelCustAicpuSo(const string &kernel_name) { } Status ModelManager::ClearAicpuSo() { - GE_CHK_STATUS_RET(LaunchKernelCustAicpuSo(kDeleteCustOp), "delete cust op so failed."); + GE_CHK_STATUS_RET(LaunchKernelCustAicpuSo(kDeleteCustOp), + "[Call][LaunchKernelCustAicpuSo] delete cust op so failed."); return SUCCESS; } Status ModelManager::LaunchCustAicpuSo() { - GE_CHK_STATUS_RET(LaunchKernelCustAicpuSo(kBatchLoadBuf), "launch cust op so failed."); + GE_CHK_STATUS_RET(LaunchKernelCustAicpuSo(kBatchLoadBuf), + "[Call][LaunchKernelCustAicpuSo] launch cust op so failed."); return SUCCESS; } @@ -1577,21 +1506,21 @@ Status ModelManager::GetModelMemAndWeightSize(const ModelData &model, size_t &me uint8_t *model_data = nullptr; uint32_t model_len = 0; Status ret = ModelParserBase::ParseModelContent(model, model_data, model_len); - GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, return ACL_ERROR_GE_PARAM_INVALID, "parse model content failed!"); + GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, return ACL_ERROR_GE_PARAM_INVALID, "[Parse][ModelContent] failed!"); OmFileLoadHelper om_file_helper; ret = om_file_helper.Init(model_data, model_len); - GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, return ret, "om file helperInit failed!"); + GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, return ret, "[Init][OmFileHelper] failed, ret:%d", ret); auto partition_table = reinterpret_cast(model_data); if (partition_table->num == 1) { REPORT_INNER_ERROR("E19999", "partition_table num in model_data is 1, check invalid"); - GELOGE(ACL_ERROR_GE_PARAM_INVALID, "om model is error,please use executable om model"); + GELOGE(ACL_ERROR_GE_PARAM_INVALID, "[Check][Param] om model is error, please use executable om model"); return ACL_ERROR_GE_PARAM_INVALID; } ModelPartition task_partition; if (om_file_helper.GetModelPartition(ModelPartitionType::TASK_INFO, task_partition) != SUCCESS) { - GELOGE(ACL_ERROR_GE_EXEC_LOAD_TASK_PARTITION_FAILED, "get task model partition failed."); + GELOGE(ACL_ERROR_GE_EXEC_LOAD_TASK_PARTITION_FAILED, "[Get][ModelPartition] failed."); return ACL_ERROR_GE_EXEC_LOAD_TASK_PARTITION_FAILED; } @@ -1601,7 +1530,7 @@ Status ModelManager::GetModelMemAndWeightSize(const ModelData &model, size_t &me } if (task_partition.size != 0) { if (!ReadProtoFromArray(task_partition.data, static_cast(task_partition.size), model_task_def.get())) { - GELOGE(ACL_ERROR_GE_EXEC_LOAD_TASK_PARTITION_FAILED, "ReadProtoFromArray failed."); + GELOGE(ACL_ERROR_GE_EXEC_LOAD_TASK_PARTITION_FAILED, "[Read][Proto] From Array failed."); return ACL_ERROR_GE_EXEC_LOAD_TASK_PARTITION_FAILED; } } @@ -1609,7 +1538,7 @@ Status ModelManager::GetModelMemAndWeightSize(const ModelData &model, size_t &me ModelPartition partition_weight; ret = om_file_helper.GetModelPartition(ModelPartitionType::WEIGHTS_DATA, partition_weight); GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, return ACL_ERROR_GE_EXEC_LOAD_WEIGHT_PARTITION_FAILED, - "Get weight partition failed. ret = %u", ret); + "[Get][ModelPartition] failed. ret = %u", ret); mem_size = model_task_def->memory_size(); weight_size = partition_weight.size; @@ -1627,8 +1556,7 @@ void ModelManager::GenModelId(uint32_t *id) { Status ModelManager::GetOrigInputInfo(uint32_t model_id, uint32_t index, OriginInputInfo &orig_input_info) { std::shared_ptr davinci_model = GetModel(model_id); GE_CHK_BOOL_RET_STATUS(davinci_model != nullptr, ACL_ERROR_GE_EXEC_MODEL_ID_INVALID, - "GetOrigInputInfo failed, invalid model_id is %u.", - model_id); + "[Get][Model] failed, invalid model_id is %u.", model_id); return davinci_model->GetOrigInputInfo(index, orig_input_info); } @@ -1638,7 +1566,7 @@ Status ModelManager::GetAllAippInputOutputDims(uint32_t model_id, uint32_t index std::vector &output_dims) { std::shared_ptr davinci_model = GetModel(model_id); GE_CHK_BOOL_RET_STATUS(davinci_model != nullptr, ACL_ERROR_GE_EXEC_MODEL_ID_INVALID, - "GetAllAippInputOutputDims failed, invalid model_id is %u.", model_id); + "[Get][Model] failed, invalid model_id is %u.", model_id); return davinci_model->GetAllAippInputOutputDims(index, input_dims, output_dims); } @@ -1653,7 +1581,7 @@ ge::Status ModelManager::SyncExecuteModel(uint32_t model_id, const vector &options dump_exception_flag_ = true; rtError_t rt_ret = rtSetTaskFailCallback(reinterpret_cast(ExceptionCallback)); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtSetTaskFailCallback fail, ret = 0x%X", - rt_ret); - GELOGE(RT_FAILED, "rtSetTaskFailCallback failed"); + REPORT_CALL_ERROR("E19999", "Call rtSetTaskFailCallback fail, ret = 0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtSetTaskFailCallback] fail, ret = 0x%X", rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } } else { @@ -1743,9 +1670,8 @@ Status ModelManager::LaunchKernelCheckAicpuOp(std::vector &aicpu_op // malloc sysOpInfoList in SysOpCheckInfo status = rtMalloc(&d_req_op_list, op_nums * sizeof(SysOpInfo), RT_MEMORY_HBM); if (status != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%zu, ret = 0x%X", - op_nums * sizeof(SysOpInfo), status); - GELOGE(RT_FAILED, "Call rt failed, status: 0x%x", status); + REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%zu, ret = 0x%X", op_nums * sizeof(SysOpInfo), status); + GELOGE(RT_FAILED, "[Call][RtMalloc] fail, size:%zu, ret = 0x%X", op_nums * sizeof(SysOpInfo), status); return RT_ERROR_TO_GE_STATUS(status); } allocated_mem.push_back(d_req_op_list); @@ -1753,9 +1679,8 @@ Status ModelManager::LaunchKernelCheckAicpuOp(std::vector &aicpu_op // malloc sysOpInfoList in SysOpCheckResp status = rtMalloc(&d_res_op_list, op_nums * sizeof(SysOpInfo), RT_MEMORY_HBM); if (status != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%zu, ret = 0x%X", - op_nums * sizeof(SysOpInfo), status); - GELOGE(RT_FAILED, "Call rt failed, status: 0x%x", status); + REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%zu, ret = 0x%X", op_nums * sizeof(SysOpInfo), status); + GELOGE(RT_FAILED, "[Call][RtMalloc] fail, size:%zu, ret = 0x%X", op_nums * sizeof(SysOpInfo), status); return RT_ERROR_TO_GE_STATUS(status); } allocated_mem.push_back(d_res_op_list); @@ -1763,9 +1688,8 @@ Status ModelManager::LaunchKernelCheckAicpuOp(std::vector &aicpu_op // malloc returnCodeList in SysOpCheckResp status = rtMalloc(&d_ret_code_list, op_nums * sizeof(ReturnCode), RT_MEMORY_HBM); if (status != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%zu, ret = 0x%X", - op_nums * sizeof(ReturnCode), status); - GELOGE(RT_FAILED, "Call rt failed, status: 0x%x", status); + REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%zu, ret = 0x%X", op_nums * sizeof(ReturnCode), status); + GELOGE(RT_FAILED, "[Call][RtMalloc] fail, size:%zu, ret = 0x%X", op_nums * sizeof(ReturnCode), status); return RT_ERROR_TO_GE_STATUS(status); } allocated_mem.push_back(d_ret_code_list); @@ -1776,9 +1700,8 @@ Status ModelManager::LaunchKernelCheckAicpuOp(std::vector &aicpu_op void *d_op_type_name = nullptr; status = rtMalloc(&d_op_type_name, op_type.length(), RT_MEMORY_HBM); if (status != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%lu, ret = 0x%X", - op_type.length(), status); - GELOGE(RT_FAILED, "Call rt failed, status: 0x%x", status); + REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%lu, ret = 0x%X", op_type.length(), status); + GELOGE(RT_FAILED, "[Call][RtMalloc] fail, size:%lu, ret = 0x%X", op_type.length(), status); return RT_ERROR_TO_GE_STATUS(status); } allocated_mem.push_back(d_op_type_name); @@ -1795,9 +1718,8 @@ Status ModelManager::LaunchKernelCheckAicpuOp(std::vector &aicpu_op void *d_op_type_name = nullptr; status = rtMalloc(&d_op_type_name, op_type.size(), RT_MEMORY_HBM); if (status != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%lu, ret = 0x%X", - op_type.length(), status); - GELOGE(RT_FAILED, "Call rt failed, status: 0x%x", status); + REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%lu, ret = 0x%X", op_type.length(), status); + GELOGE(RT_FAILED, "[Call][RtMalloc] fail, size:%lu, ret = 0x%X", op_type.size(), status); return RT_ERROR_TO_GE_STATUS(status); } allocated_mem.push_back(d_op_type_name); @@ -1825,9 +1747,8 @@ Status ModelManager::LaunchKernelCheckAicpuOp(std::vector &aicpu_op uint32_t args_size = sizeof(SysOpCheckInfo) + sizeof(SysOpCheckResp); status = rtMalloc(&args, args_size, RT_MEMORY_HBM); if (status != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%u, ret = 0x%X", - args_size, status); - GELOGE(RT_FAILED, "Call rt failed, status: 0x%x", status); + REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, size:%u, ret = 0x%X", args_size, status); + GELOGE(RT_FAILED, "[Call][RtMalloc] fail, size:%u, ret = 0x%X", args_size, status); return RT_ERROR_TO_GE_STATUS(status); } allocated_mem.push_back(args); @@ -1842,9 +1763,8 @@ Status ModelManager::LaunchKernelCheckAicpuOp(std::vector &aicpu_op status = rtStreamSynchronize(stream); if (status != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtStreamSynchronize fail, ret = 0x%X", - status); - GELOGE(RT_FAILED, "Call rt stream sync failed, status: 0x%x", status); + REPORT_CALL_ERROR("E19999", "Call rtStreamSynchronize fail, ret = 0x%X", status); + GELOGE(RT_FAILED, "[Call][RtStreamSynchronize] failed, ret:0x%X", status); GE_CHK_RT(rtStreamDestroy(stream)); return RT_ERROR_TO_GE_STATUS(status); } @@ -1879,7 +1799,8 @@ Status ModelManager::LaunchKernelCheckAicpuOp(std::vector &aicpu_op REPORT_INNER_ERROR("E19999", "res_ret_code_list.size:%zu res_aicpu_op_info_list.size:%zu res_op_nums:%lu " "not equal, check invalid", res_ret_code_list.size(), res_aicpu_op_info_list.size(), res_op_nums); - GELOGE(FAILED, "Number of retcode is not equal to number of op type."); + GELOGE(FAILED, "[Check][Param] Number:%zu of retcode is not equal to number:%zu of op type or not equal %lu.", + res_ret_code_list.size(), res_aicpu_op_info_list.size(), res_op_nums); GE_CHK_RT(rtStreamDestroy(stream)); return FAILED; } @@ -1902,9 +1823,8 @@ Status ModelManager::LaunchKernelCheckAicpuOp(std::vector &aicpu_op "<0: op_type, 1: format, 2: datatype> \n"; } fail_reason += "not support."; - REPORT_INNER_ERROR("E19999", "Check aicpu op_type failed, details:%s", - fail_reason.c_str()); - GELOGE(FAILED, "Check aicpu op_type failed. details: %s", fail_reason.c_str()); + REPORT_INNER_ERROR("E19999", "Check aicpu op_type failed, details:%s", fail_reason.c_str()); + GELOGE(FAILED, "[Check][Param] Check aicpu op_type failed. details:%s", fail_reason.c_str()); GE_CHK_RT(rtStreamDestroy(stream)); return FAILED; } @@ -1924,7 +1844,7 @@ Status ModelManager::CheckAicpuOpList(GeModelPtr ge_model) { return SUCCESS; } GE_CHK_STATUS_RET(LaunchKernelCheckAicpuOp(aicpu_optype_list, aicpu_tf_optype_list), - "Launch check aicpu op type failed."); + "[Call][LaunchKernelCheckAicpuOp] failed."); return SUCCESS; } diff --git a/ge/graph/load/model_manager/model_manager.h b/ge/graph/load/model_manager/model_manager.h index c0f14934..e35bb7aa 100755 --- a/ge/graph/load/model_manager/model_manager.h +++ b/ge/graph/load/model_manager/model_manager.h @@ -310,7 +310,7 @@ class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ModelManager { std::lock_guard lock(exeception_infos_mutex_); auto instance = ModelManager::GetInstance(); if (instance == nullptr) { - GELOGE(FAILED, "Instance is nullptr"); + GELOGE(FAILED, "[Get][Instance] failed, as ret is nullptr"); return; } instance->AddExceptionInfo(*rt_exception_info); @@ -345,17 +345,6 @@ class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ModelManager { void GenModelId(uint32_t *id); - Status InitDumPropertiesWithNewSessionId(uint64_t session_id); - - bool IsDumpSeverInited(uint64_t session_id); - - Status AddDumpProperties(uint64_t session_id, const DumpProperties &dump_properties); - - Status UpdateSessionId(uint32_t model_id, GeModelPtr ge_model, - std::shared_ptr &davinci_model, uint64_t &session_id); - - bool HasVarNode(ComputeGraphPtr &compute_graph) const; - std::map> model_map_; std::map> hybrid_model_map_; std::map> model_aicpu_kernel_; diff --git a/ge/graph/load/model_manager/model_utils.cc b/ge/graph/load/model_manager/model_utils.cc index f6ff591a..c15c6872 100755 --- a/ge/graph/load/model_manager/model_utils.cc +++ b/ge/graph/load/model_manager/model_utils.cc @@ -22,16 +22,25 @@ #include "graph/manager/graph_var_manager.h" #include "graph/types.h" #include "graph/build/memory/block_mem_assigner.h" - -#define VALIDATE_MEM_RANGE(OP, SIZE, OFFSET) \ - do { \ - if (SIZE <= static_cast(OFFSET)) { \ - REPORT_INNER_ERROR("E19999", \ - "Node:%s(%s) offset:%ld out of range size:%lu, check invalid", \ - OP->GetName().c_str(), OP->GetType().c_str(), OFFSET, SIZE); \ - GELOGE(OUT_OF_MEMORY, "Node: %s, memory out of range[%lu: %ld]", OP->GetName().c_str(), SIZE, OFFSET); \ - return {}; \ - } \ +#include "common/math/math_util.h" + +#define VALIDATE_MEM_RANGE(OP, TOTAL_SIZE, OFFSET, SIZE) \ + do { \ + if (ge::CheckInt64AddOverflow((OFFSET), (SIZE)) != SUCCESS) { \ + GELOGE(PARAM_INVALID, "Int64 %ld and %ld addition can result in overflow!", \ + static_cast(OFFSET), static_cast(SIZE)); \ + return {}; \ + } \ + int64_t range = (OFFSET) + (SIZE); \ + if ((TOTAL_SIZE) < static_cast(range)) { \ + REPORT_INNER_ERROR("E19999", \ + "Node:%s(%s) memory out of range, offset:%ld, size:%ld, exceed total size:%lu.", \ + OP->GetName().c_str(), OP->GetType().c_str(), (OFFSET), (SIZE), (TOTAL_SIZE)); \ + GELOGE(OUT_OF_MEMORY, \ + "[Check][Param]Node:%s(%s) memory out of range, offset:%ld, size:%ld, exceed total size:%lu.", \ + OP->GetName().c_str(), OP->GetType().c_str(), (OFFSET), (SIZE), (TOTAL_SIZE)); \ + return {}; \ + } \ } while (0) namespace ge { @@ -312,27 +321,30 @@ vector ModelUtils::GetInputDataAddrs(const RuntimeParam &model_param, Co REPORT_INNER_ERROR("E19999", "Attr:%s, memory_type.size:%zu != input_desc.size:%zu, op:%s(%s), check invalid", ATTR_NAME_INPUT_MEM_TYPE_LIST.c_str(), v_memory_type.size(), inputs_size, op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(PARAM_INVALID, "Fusion: check input size failed, op: %s, input v_memory_type size: %zu input numbers: %zu", - op_desc->GetName().c_str(), v_memory_type.size(), inputs_size); + GELOGE(PARAM_INVALID, "[Check][Param] Attr:%s, memory_type.size:%zu != input_desc.size:%zu, op:%s(%s)", + ATTR_NAME_INPUT_MEM_TYPE_LIST.c_str(), v_memory_type.size(), inputs_size, + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return v_input_data_addr; } for (size_t i = 0; i < op_desc->GetAllInputsSize(); ++i) { const GeTensorDescPtr tensor_desc = op_desc->MutableInputDesc(static_cast(i)); GE_IF_BOOL_EXEC(tensor_desc == nullptr, GELOGD("Op: %s, Index: %zu, has no input", op_desc->GetName().c_str(), i); continue;) + int64_t tensor_size = 0; + GE_CHK_STATUS_EXEC(TensorUtils::GetSize(*tensor_desc, tensor_size), return {}); if ((i < v_is_input_const.size()) && v_is_input_const[i]) { - // TBE: add weights address to input - int64_t tensor_size = 0; - GE_CHK_STATUS(TensorUtils::GetSize(*tensor_desc, tensor_size)); - if (tensor_size) { - int64_t data_offset = 0; - GE_CHK_STATUS(TensorUtils::GetDataOffset(*tensor_desc, data_offset)); - VALIDATE_MEM_RANGE(op_desc, model_param.weight_size, data_offset); - uint8_t *weight_addr = model_param.weight_base + data_offset; - v_input_data_addr.push_back(weight_addr); - GELOGI("[IMAS]GetInputDataAddrs graph_%u type[C] name[%s] input[%zu] memaddr[%p]", model_param.graph_id, - op_desc->GetName().c_str(), i, weight_addr); - } + // Add weights address to input + int64_t data_offset = 0; + GE_CHK_STATUS(TensorUtils::GetDataOffset(*tensor_desc, data_offset)); + int64_t weight_size = 0; + // The reason why GetTensorSizeInBytes is used here is that the weight is allocated based on the size of + // TensorData in function AdjustConstWeightSize. and the size is zero when the tensor is empty. + GE_CHK_STATUS(TensorUtils::GetTensorSizeInBytes(*tensor_desc, weight_size)); + VALIDATE_MEM_RANGE(op_desc, model_param.weight_size, data_offset, weight_size); + uint8_t *weight_addr = model_param.weight_base + data_offset; + v_input_data_addr.push_back(weight_addr); + GELOGI("[IMAS]GetInputDataAddrs graph_%u type[C] name[%s] input[%zu] memaddr[%p]", model_param.graph_id, + op_desc->GetName().c_str(), i, weight_addr); non_const_index++; continue; } @@ -341,9 +353,13 @@ vector ModelUtils::GetInputDataAddrs(const RuntimeParam &model_param, Co int64_t input_offset = v_input_offset[non_const_index]; non_const_index++; - GE_IF_BOOL_EXEC(model_param.var_size != 0 && ge::VarManager::Instance(session_id)->IsVarAddr(input_offset), + int64_t inner_offset = 0; + (void)ge::AttrUtils::GetInt(op_desc->MutableInputDesc(i), ATTR_NAME_INNER_OFFSET, inner_offset); + GE_IF_BOOL_EXEC(model_param.var_size != 0 && ge::VarManager::Instance(session_id)->IsVarAddr(input_offset - inner_offset), uint8_t *variable_addr = nullptr; - GE_CHK_STATUS_EXEC(GetVarAddr(model_param, op_desc, input_offset, variable_addr), return {}); + GE_CHK_STATUS_EXEC(GetVarAddr(model_param, op_desc, input_offset - inner_offset, + tensor_size + inner_offset, variable_addr), return {}); + variable_addr += inner_offset; v_input_data_addr.push_back(variable_addr); GELOGI("[IMAS]GetInputDataAddrs graph_%u type[V] name[%s] input[%lu] memaddr[%p]", model_param.graph_id, op_desc->GetName().c_str(), i, variable_addr); @@ -357,9 +373,8 @@ vector ModelUtils::GetInputDataAddrs(const RuntimeParam &model_param, Co mem_addr = reinterpret_cast(static_cast(input_offset)); v_input_data_addr.push_back(mem_addr); } else if (has_mem_type_attr && v_memory_type[i] == RT_MEMORY_TS_4G) { - int64_t tensor_size = 0; - GE_CHK_STATUS_EXEC(TensorUtils::GetSize(*tensor_desc, tensor_size), return {}); - VALIDATE_MEM_RANGE(op_desc, model_param.mem_size, input_offset); + // The input size and peer output size may be not consecutive, therefore, the tensor_size is not been checked. + VALIDATE_MEM_RANGE(op_desc, model_param.mem_size, input_offset, static_cast(0)); mem_addr = model_param.ts_mem_mall->Acquire(input_offset, static_cast(tensor_size)); v_input_data_addr.push_back(mem_addr); } else if (tensor_has_mem_type && mem_type == RT_MEMORY_P2P_DDR) { @@ -369,7 +384,8 @@ vector ModelUtils::GetInputDataAddrs(const RuntimeParam &model_param, Co op_desc->GetName().c_str(), i, p2p_mem_addr); continue; } else { - VALIDATE_MEM_RANGE(op_desc, model_param.mem_size, input_offset); + // The input size and peer output size may be not consecutive, therefore, the tensor_size is not been checked. + VALIDATE_MEM_RANGE(op_desc, model_param.mem_size, input_offset, static_cast(0)); mem_addr = model_param.mem_base + input_offset; v_input_data_addr.push_back(mem_addr); } @@ -386,26 +402,25 @@ vector ModelUtils::GetInputDataAddrs(const RuntimeParam &model_param, Co /// @return Status /// Status ModelUtils::GetVarAddr(const RuntimeParam &model_param, const ConstOpDescPtr &op_desc, int64_t offset, - uint8_t *&var_addr) { + int64_t tensor_size, uint8_t *&var_addr) { rtMemType_t mem_type = ge::VarManager::Instance(model_param.session_id)->GetVarMemType(offset); switch (mem_type) { case RT_MEMORY_RDMA_HBM: if (offset < 0) { REPORT_INNER_ERROR("E19999", "Param offset:%ld < 0, check invalid", offset); - GELOGE(PARAM_INVALID, "rdma var addr is invalid, addr=%p", - reinterpret_cast(static_cast(offset))); + GELOGE(PARAM_INVALID, "[Check][Param] Param offset:%ld cannot be negative", offset); return PARAM_INVALID; } var_addr = reinterpret_cast(static_cast(offset)); break; case RT_MEMORY_HBM: - VALIDATE_MEM_RANGE(op_desc, model_param.var_size, offset - model_param.logic_var_base); + VALIDATE_MEM_RANGE(op_desc, model_param.var_size, offset - model_param.logic_var_base, tensor_size); var_addr = model_param.var_base + offset - model_param.logic_var_base; break; default: - REPORT_INNER_ERROR("E19999", "Get mem_type:%d for offset:%ld is unsupported, check invalid", - mem_type, offset); - GELOGE(PARAM_INVALID, "unsupported memory type %u", mem_type); + REPORT_INNER_ERROR("E19999", "Get mem_type:%d for offset:%ld is unsupported, check invalid", mem_type, offset); + GELOGE(PARAM_INVALID, "[Check][Param] Get mem_type:%d for offset:%ld is unsupported, check invalid", + mem_type, offset); return PARAM_INVALID; } GE_CHECK_NOTNULL(var_addr); @@ -433,9 +448,9 @@ vector ModelUtils::GetOutputDataAddrs(const RuntimeParam &model_param, C REPORT_INNER_ERROR("E19999", "Attr:%s, memory_type.size:%zu != output_desc.size:%zu, op:%s(%s), check invalid", ATTR_NAME_OUTPUT_MEM_TYPE_LIST.c_str(), v_memory_type.size(), outputs_size, op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(PARAM_INVALID, - "Fusion: check output size failed, op: %s, output v_memory_type size: %lu output numbers: %zu", - op_desc->GetName().c_str(), v_memory_type.size(), outputs_size); + GELOGE(PARAM_INVALID, "[Check][Param] Attr:%s, memory_type.size:%zu != output_desc.size:%zu, op:%s(%s)", + ATTR_NAME_OUTPUT_MEM_TYPE_LIST.c_str(), v_memory_type.size(), outputs_size, + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return v_output_data_addr; } for (size_t i = 0; i < outputs_size; ++i) { @@ -451,9 +466,15 @@ vector ModelUtils::GetOutputDataAddrs(const RuntimeParam &model_param, C GELOGD("%s is an optional output, the address don't need to be saved.", tensor_desc->GetName().c_str()); continue; } - GE_IF_BOOL_EXEC(model_param.var_size != 0 && ge::VarManager::Instance(session_id)->IsVarAddr(v_output_offset[i]), + int64_t inner_offset = 0; + (void)ge::AttrUtils::GetInt(op_desc->MutableOutputDesc(i), ATTR_NAME_INNER_OFFSET, inner_offset); + int64_t tensor_size = 0; + GE_CHK_STATUS_EXEC(TensorUtils::GetSize(*tensor_desc, tensor_size), return {}); + GE_IF_BOOL_EXEC(model_param.var_size != 0 && ge::VarManager::Instance(session_id)->IsVarAddr(v_output_offset[i] - inner_offset), uint8_t *variable_addr = nullptr; - GE_CHK_STATUS_EXEC(GetVarAddr(model_param, op_desc, v_output_offset[i], variable_addr), return {}); + GE_CHK_STATUS_EXEC(GetVarAddr(model_param, op_desc, v_output_offset[i] - inner_offset, + tensor_size + inner_offset, variable_addr), return {}); + variable_addr += inner_offset; v_output_data_addr.push_back(variable_addr); GELOGI("[IMAS]GetOutputDataAddrs graph_%u type[V] name[%s] output[%zu] memaddr[%p]", model_param.graph_id, op_desc->GetName().c_str(), i, variable_addr); @@ -467,11 +488,7 @@ vector ModelUtils::GetOutputDataAddrs(const RuntimeParam &model_param, C mem_addr = reinterpret_cast(static_cast(v_output_offset[i])); v_output_data_addr.push_back(mem_addr); } else if (has_mem_type_attr && v_memory_type[i] == RT_MEMORY_TS_4G) { - const GeTensorDescPtr tensor_desc = op_desc->MutableOutputDesc(i); - GE_CHECK_NOTNULL_EXEC(tensor_desc, return {}); - int64_t tensor_size = 0; - GE_CHK_STATUS_EXEC(TensorUtils::GetSize(*tensor_desc, tensor_size), return {}); - VALIDATE_MEM_RANGE(op_desc, model_param.mem_size, v_output_offset[i]); + VALIDATE_MEM_RANGE(op_desc, model_param.mem_size, v_output_offset[i], tensor_size); mem_addr = model_param.ts_mem_mall->Acquire(v_output_offset[i], static_cast(tensor_size)); v_output_data_addr.push_back(mem_addr); } else if (tensor_has_mem_type && mem_type == RT_MEMORY_P2P_DDR) { @@ -481,7 +498,7 @@ vector ModelUtils::GetOutputDataAddrs(const RuntimeParam &model_param, C op_desc->GetName().c_str(), i, p2p_mem_addr); continue; } else { - VALIDATE_MEM_RANGE(op_desc, model_param.mem_size, v_output_offset[i]); + VALIDATE_MEM_RANGE(op_desc, model_param.mem_size, v_output_offset[i], tensor_size); mem_addr = static_cast(model_param.mem_base + v_output_offset[i]); v_output_data_addr.push_back(mem_addr); } @@ -554,7 +571,7 @@ vector ModelUtils::GetWorkspaceDataAddrs(const RuntimeParam &model_param GELOGI("[IMAS]GetWorkspaceDataAddrs graph_%u type[F] name[%s] workspace[%zu] offset[%ld] bytes[%ld] Null addr", model_param.graph_id, op_desc->GetName().c_str(), i, v_workspace_offset[i], v_workspace_bytes[i]); } else { - VALIDATE_MEM_RANGE(op_desc, model_param.mem_size, v_workspace_offset[i]); + VALIDATE_MEM_RANGE(op_desc, model_param.mem_size, v_workspace_offset[i], v_workspace_bytes[i]); uint8_t *mem_addr = nullptr; bool session_scope_memory = (has_workspace_no_reuse_scope) && (i < workspace_no_reuse_scope.size()); if (session_scope_memory) { @@ -594,7 +611,7 @@ Status ModelUtils::GetRtAddress(const RuntimeParam ¶m, uintptr_t logic_addr, } else if (logic_addr != 0) { mem_addr = nullptr; REPORT_INNER_ERROR("E19999", "Check param logic addr:0x%lx abnormal", logic_addr); - GELOGE(PARAM_INVALID, "The logic addr:0x%lx is abnormal", logic_addr); + GELOGE(PARAM_INVALID, "[Check][Param] The logic addr:0x%lx is abnormal", logic_addr); return PARAM_INVALID; } diff --git a/ge/graph/load/model_manager/model_utils.h b/ge/graph/load/model_manager/model_utils.h index 26f8d700..8ce1b060 100755 --- a/ge/graph/load/model_manager/model_utils.h +++ b/ge/graph/load/model_manager/model_utils.h @@ -115,7 +115,7 @@ class ModelUtils { /// @return Status /// static Status GetVarAddr(const RuntimeParam &model_param, const ConstOpDescPtr &op_desc, int64_t offset, - uint8_t *&var_addr); + int64_t tensor_size, uint8_t *&var_addr); }; } // namespace ge diff --git a/ge/graph/load/model_manager/task_info/kernel_ex_task_info.cc b/ge/graph/load/model_manager/task_info/kernel_ex_task_info.cc index 7c26d23d..356919f6 100644 --- a/ge/graph/load/model_manager/task_info/kernel_ex_task_info.cc +++ b/ge/graph/load/model_manager/task_info/kernel_ex_task_info.cc @@ -195,7 +195,8 @@ Status KernelExTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davin return RT_ERROR_TO_GE_STATUS(rt_ret);) SetIoAddrs(op_desc); - InitDumpTask(input_output_addr, op_desc); + InitDumpFlag(op_desc); + InitDumpArgs(input_output_addr, op_desc); GELOGI("KernelExTaskInfo knonw node Init Success."); return SUCCESS; } @@ -237,7 +238,8 @@ Status KernelExTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davin GELOGE(RT_FAILED, "[Call][RtMemcpy] failed, ret:0x%X, size:%lu", rt_ret, addrs_size); return RT_ERROR_TO_GE_STATUS(rt_ret);) - InitDumpTask(input_output_addr_, op_desc); + InitDumpFlag(op_desc); + InitDumpArgs(input_output_addr_, op_desc); } uint64_t input_output_addr = static_cast(reinterpret_cast(input_output_addr_)); @@ -269,10 +271,16 @@ Status KernelExTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davin return SUCCESS; } -void KernelExTaskInfo::InitDumpTask(void *addr, const OpDescPtr &op_desc) { - if (davinci_model_->OpNeedDump(op_desc->GetName()) || davinci_model_->GetOpDugReg()) { - GELOGD("Op %s need dump in kernel ex task info", op_desc->GetName().c_str()); +void KernelExTaskInfo::InitDumpFlag(const OpDescPtr &op_desc) { + if (davinci_model_->OpNeedDump(op_desc->GetName())) { + GELOGD("Op %s need init dump flag in kernel ex task info", op_desc->GetName().c_str()); dump_flag_ = RT_KERNEL_DUMPFLAG; + } +} + +void KernelExTaskInfo::InitDumpArgs(void *addr, const OpDescPtr &op_desc) { + if (davinci_model_->OpNeedDump(op_desc->GetName())) { + GELOGD("Op %s need dump in kernel ex task info", op_desc->GetName().c_str()); dump_args_ = addr; } if (davinci_model_->GetOpDugReg()) { diff --git a/ge/graph/load/model_manager/task_info/kernel_ex_task_info.h b/ge/graph/load/model_manager/task_info/kernel_ex_task_info.h index bcc17168..1b77b715 100644 --- a/ge/graph/load/model_manager/task_info/kernel_ex_task_info.h +++ b/ge/graph/load/model_manager/task_info/kernel_ex_task_info.h @@ -61,7 +61,8 @@ class KernelExTaskInfo : public TaskInfo { Status CopyTaskInfo(const domi::KernelExDef &kernel_def, const RuntimeParam &rts_param, const OpDescPtr &op_desc); void SetIoAddrs(const OpDescPtr &op_desc); - void InitDumpTask(void *addr, const OpDescPtr &op_desc); + void InitDumpFlag(const OpDescPtr &op_desc); + void InitDumpArgs(void *addr, const OpDescPtr &op_desc); Status InitTaskExtInfo(const std::string &ext_info, const OpDescPtr &op_desc); uint32_t task_id_; diff --git a/ge/graph/load/model_manager/task_info/kernel_task_info.cc b/ge/graph/load/model_manager/task_info/kernel_task_info.cc index c72bfeef..919a56cd 100755 --- a/ge/graph/load/model_manager/task_info/kernel_task_info.cc +++ b/ge/graph/load/model_manager/task_info/kernel_task_info.cc @@ -129,6 +129,7 @@ Status KernelTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davinci ctx_.opIndex2[i] = context.origin_op_index(i); } ctx_.opCount = context.origin_op_index_size(); + InitDumpFlag(); if (kernel_type_ == ccKernelType::TE) { ctx_.opIndex = context.op_index(); uint16_t *args_offset_tmp = reinterpret_cast(const_cast(context.args_offset().data())); @@ -660,7 +661,7 @@ Status KernelTaskInfo::InitTVMTask(uint16_t offset, const domi::KernelDef &kerne if (davinci_model_->IsKnownNode()) { args_ = l2_buffer_on_ ? davinci_model_->GetCurrentHybridArgsAddr(hybrid_args_offset_) : davinci_model_->GetCurrentArgsAddr(args_offset_); - InitDumpTask(offset); + InitDumpArgs(offset); return SUCCESS; } @@ -726,7 +727,7 @@ Status KernelTaskInfo::InitTVMTask(uint16_t offset, const domi::KernelDef &kerne return FAILED; } skt_dump_args_ = static_cast(args_) + offset; - InitDumpTask(offset); + InitDumpArgs(offset); vector virtual_io_addrs; // use virtual address for zero copy key. virtual_io_addrs.insert(virtual_io_addrs.end(), input_data_addrs.begin(), input_data_addrs.end()); @@ -1022,7 +1023,7 @@ Status KernelTaskInfo::InitAicpuTask(uint32_t op_index, const domi::KernelDef &k if (davinci_model_->IsKnownNode()) { args_ = davinci_model_->GetCurrentHybridArgsAddr(hybrid_args_offset_); - InitDumpTask(sizeof(aicpu::AicpuParamHead)); + InitDumpArgs(sizeof(aicpu::AicpuParamHead)); return SUCCESS; } const RuntimeParam &rts_param = davinci_model_->GetRuntimeParam(); @@ -1063,31 +1064,36 @@ Status KernelTaskInfo::InitAicpuTask(uint32_t op_index, const domi::KernelDef &k op_desc->GetName().c_str(), op_desc->GetType().c_str(), args_size_, rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } - InitDumpTask(sizeof(aicpu::AicpuParamHead)); - - if (kernel_type_ == ccKernelType::CUST_AI_CPU) { - dump_flag_ |= RT_KERNEL_CUSTOM_AICPU; - } + InitDumpArgs(sizeof(aicpu::AicpuParamHead)); davinci_model_->SetZeroCopyAddr(op_desc, io_addrs, args_addr.get(), args_, args_size_, sizeof(aicpu::AicpuParamHead)); return SUCCESS; } -void KernelTaskInfo::InitDumpTask(uint32_t offset) { +void KernelTaskInfo::InitDumpFlag() { if (davinci_model_->OpNeedDump(op_desc_->GetName())) { - GELOGD("Op %s need dump in task info", op_desc_->GetName().c_str()); + GELOGD("Op %s init dump flag", op_desc_->GetName().c_str()); if (IsL1FusionOp(op_desc_)) { dump_flag_ = RT_FUSION_KERNEL_DUMPFLAG; } else { dump_flag_ = RT_KERNEL_DUMPFLAG; } + } +} + +void KernelTaskInfo::InitDumpArgs(uint32_t offset) { + if (davinci_model_->OpNeedDump(op_desc_->GetName())) { + GELOGD("Op %s need dump in task info", op_desc_->GetName().c_str()); dump_args_ = static_cast(args_) + offset; } if (davinci_model_->GetOpDugReg()) { GELOGD("Op debug is open in kernel task info"); dump_args_ = static_cast(args_) + offset; } + if (kernel_type_ == ccKernelType::CUST_AI_CPU) { + dump_flag_ |= RT_KERNEL_CUSTOM_AICPU; + } } Status KernelTaskInfo::InitAicpuTaskExtInfo(const std::string &ext_info) { diff --git a/ge/graph/load/model_manager/task_info/kernel_task_info.h b/ge/graph/load/model_manager/task_info/kernel_task_info.h index 79347255..d9dd30bb 100644 --- a/ge/graph/load/model_manager/task_info/kernel_task_info.h +++ b/ge/graph/load/model_manager/task_info/kernel_task_info.h @@ -128,7 +128,8 @@ class KernelTaskInfo : public TaskInfo { Status SuperKernelDistribute(); bool IsL1FusionOp(const OpDescPtr &op_desc); void SetIoAddrs(const OpDescPtr &op_desc); - void InitDumpTask(uint32_t offset); + void InitDumpFlag(); + void InitDumpArgs(uint32_t offset); void SetContinuousArgs(uint32_t args_size, DavinciModel *davinci_model); void SetNoncontinuousArgs(uint32_t args_size, DavinciModel *davinci_model); Status CopyNoncontinuousArgs(uint16_t offset); diff --git a/ge/graph/load/model_manager/task_info/task_info.cc b/ge/graph/load/model_manager/task_info/task_info.cc index fad18d37..9f82efad 100755 --- a/ge/graph/load/model_manager/task_info/task_info.cc +++ b/ge/graph/load/model_manager/task_info/task_info.cc @@ -26,7 +26,7 @@ Status TaskInfo::SetStream(uint32_t stream_id, const std::vector &st stream_ = stream_list[stream_id]; } else { REPORT_INNER_ERROR("E19999", "stream_id:%u >= stream_list.size(): %zu, check invalid", - stream_id, stream_list.size()); + stream_id, stream_list.size()); GELOGE(FAILED, "[Check][Param] index:%u >= stream_list.size():%zu.", stream_id, stream_list.size()); return FAILED; } diff --git a/ge/graph/load/model_manager/task_info/task_info.h b/ge/graph/load/model_manager/task_info/task_info.h index 5657f003..9f849b20 100644 --- a/ge/graph/load/model_manager/task_info/task_info.h +++ b/ge/graph/load/model_manager/task_info/task_info.h @@ -49,7 +49,7 @@ struct RuntimeParam { << ", label_num:" << label_num << ", logic_mem_base:" << logic_mem_base << ", logic_weight_base:" << logic_weight_base << ", logic_var_base:" << logic_var_base << ", memory_size:" << mem_size << ", weight_size:" << weight_size << ", var_size:" << var_size - << ", ex_memory_info:"; + << ", zero_copy_size:" << zero_copy_size << ", ex_memory_info:"; for (auto it : memory_infos) { ss << "[memory_type:" << it.first << ", memory_size:" << it.second.memory_size << "]"; } @@ -65,6 +65,7 @@ struct RuntimeParam { uint64_t var_size = 0; uint64_t logic_var_base = 0; uint8_t *var_base = nullptr; + int64_t zero_copy_size = 0; std::map memory_infos; uint32_t batch_num = 0; uint32_t stream_num = 0; diff --git a/ge/graph/load/model_manager/tbe_handle_store.cc b/ge/graph/load/model_manager/tbe_handle_store.cc index 6efb6190..36207aa2 100755 --- a/ge/graph/load/model_manager/tbe_handle_store.cc +++ b/ge/graph/load/model_manager/tbe_handle_store.cc @@ -24,7 +24,7 @@ namespace ge { void TbeHandleInfo::used_inc(uint32_t num) { if (used_ > std::numeric_limits::max() - num) { REPORT_INNER_ERROR("E19999", "Used:%u reach numeric max", used_); - GELOGE(INTERNAL_ERROR, "Used[%u] reach numeric max.", used_); + GELOGE(INTERNAL_ERROR, "[Check][Param] Used[%u] reach numeric max.", used_); return; } @@ -34,7 +34,7 @@ void TbeHandleInfo::used_inc(uint32_t num) { void TbeHandleInfo::used_dec(uint32_t num) { if (used_ < std::numeric_limits::min() + num) { REPORT_INNER_ERROR("E19999", "Used:%u reach numeric min", used_); - GELOGE(INTERNAL_ERROR, "Used[%u] reach numeric min.", used_); + GELOGE(INTERNAL_ERROR, "[Check][Param] Used[%u] reach numeric min.", used_); return; } @@ -107,9 +107,8 @@ void TBEHandleStore::ReferTBEHandle(const std::string &name) { std::lock_guard lock(mutex_); auto it = kernels_.find(name); if (it == kernels_.end()) { - REPORT_INNER_ERROR("E19999", "Kernel:%s not found in stored check invalid", - name.c_str()); - GELOGE(INTERNAL_ERROR, "Kernel[%s] not found in stored.", name.c_str()); + REPORT_INNER_ERROR("E19999", "Kernel:%s not found in stored check invalid", name.c_str()); + GELOGE(INTERNAL_ERROR, "[Check][Param] Kernel[%s] not found in stored.", name.c_str()); return; } @@ -128,9 +127,8 @@ void TBEHandleStore::EraseTBEHandle(const std::map &names for (auto &item : names) { auto it = kernels_.find(item.first); if (it == kernels_.end()) { - REPORT_INNER_ERROR("E19999", "Kernel:%s not found in stored check invalid", - item.first.c_str()); - GELOGE(INTERNAL_ERROR, "Kernel[%s] not found in stored.", item.first.c_str()); + REPORT_INNER_ERROR("E19999", "Kernel:%s not found in stored check invalid", item.first.c_str()); + GELOGE(INTERNAL_ERROR, "[Check][Param] Kernel[%s] not found in stored.", item.first.c_str()); continue; } @@ -142,7 +140,8 @@ void TBEHandleStore::EraseTBEHandle(const std::map &names if (rt_ret != RT_ERROR_NONE) { REPORT_INNER_ERROR("E19999", "Call rtDevBinaryUnRegister failed for Kernel:%s fail, ret:0x%X", item.first.c_str(), rt_ret); - GELOGE(INTERNAL_ERROR, "Kernel[%s] UnRegister handle fail:%u.", item.first.c_str(), rt_ret); + GELOGE(INTERNAL_ERROR, "[Call][RtDevBinaryUnRegister] Kernel[%s] UnRegister handle fail:%u.", + item.first.c_str(), rt_ret); } kernels_.erase(it); } diff --git a/ge/graph/load/model_manager/ts_mem_mall.h b/ge/graph/load/model_manager/ts_mem_mall.h index 74ce5a16..986b3101 100644 --- a/ge/graph/load/model_manager/ts_mem_mall.h +++ b/ge/graph/load/model_manager/ts_mem_mall.h @@ -43,7 +43,7 @@ class TsMemMall { for (auto it : mem_store_size_) { rtError_t ret = rtFree(it.second); if (ret != RT_ERROR_NONE) { - GELOGE(RT_FAILED, "Call rtFree failed, ret: 0x%X", ret); + GELOGE(RT_FAILED, "[Call][RtFree] failed, ret:0x%X", ret); } } mem_store_size_.clear(); @@ -52,7 +52,7 @@ class TsMemMall { void *Acquire(int64_t offset, uint64_t size) { if (size == 0) { - GELOGE(RT_FAILED, "Acquire mem block failed, size: %lu", size); + GELOGE(RT_FAILED, "[Check][Param] Acquire mem block failed, size:%lu", size); return nullptr; } @@ -71,7 +71,7 @@ class TsMemMall { void *addr = nullptr; rtError_t rt_ret = rtMalloc(&addr, bytes, mem_type_); if (rt_ret != RT_ERROR_NONE) { - GELOGE(RT_FAILED, "Call rtMalloc failed, ret: 0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtMalloc] failed, size:%lu, ret:0x%X", bytes, rt_ret); return nullptr; } @@ -94,7 +94,7 @@ class TsMemMall { mem_store_addr_.erase(it); rtError_t ret = rtFree(addr); if (ret != RT_ERROR_NONE) { - GELOGE(RT_FAILED, "Call rtFree failed, ret: 0x%X", ret); + GELOGE(RT_FAILED, "[Call][RtFree] failed, ret:0x%X", ret); } } diff --git a/ge/graph/load/model_manager/zero_copy_offset.cc b/ge/graph/load/model_manager/zero_copy_offset.cc index 9d6f4e4f..4a57a899 100644 --- a/ge/graph/load/model_manager/zero_copy_offset.cc +++ b/ge/graph/load/model_manager/zero_copy_offset.cc @@ -38,8 +38,13 @@ Status ZeroCopyOffset::InitInputDataInfo(int64_t output_size, void *virtual_addr op_name_ = op_desc->GetName(); (void)ge::AttrUtils::GetListInt(op_desc, ATTR_ZERO_COPY_BASIC_OFFSET, zero_copy_basic_offset_); (void)ge::AttrUtils::GetListInt(op_desc, ATTR_ZERO_COPY_RELATIVE_OFFSET, zero_copy_relative_offset_); - GE_CHK_BOOL_EXEC(zero_copy_basic_offset_.size() == zero_copy_relative_offset_.size(), return PARAM_INVALID, - "basic_offset_size should be equal to relative_offset_size"); + GE_CHK_BOOL_EXEC(zero_copy_basic_offset_.size() == zero_copy_relative_offset_.size(), + REPORT_INNER_ERROR("E19999", "basic_offset_size:%zu not equal to relative_offset_size:%zu, " + "check invalid", zero_copy_basic_offset_.size(), + zero_copy_relative_offset_.size()); + return PARAM_INVALID, + "[Check][Param] basic_offset_size:%zu should be equal to relative_offset_size:%zu", + zero_copy_basic_offset_.size(), zero_copy_relative_offset_.size()); GELOGD("[ZCPY] zero_copy_basic_offset size is %zu", zero_copy_basic_offset_.size()); int64_t virtual_addr_offset = op_desc->GetOutputOffset().at(kDataIndex); @@ -78,7 +83,8 @@ Status ZeroCopyOffset::InitOutputDataInfo(const vector &input_size_list if (TensorUtils::GetTensorSizeInBytes(*tensor_desc, size) != GRAPH_SUCCESS) { REPORT_INNER_ERROR("E19999", "Get input TensorSize in op:%s(%s) failed, input_index:%zu", op_desc->GetName().c_str(), op_desc->GetType().c_str(), idx); - GELOGE(FAILED, "GetTensorSizeInBytes failed!"); + GELOGE(FAILED, "[Get][InputTensorSize] in op:%s(%s) failed, input_index:%zu", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), idx); return FAILED; } @@ -88,8 +94,13 @@ Status ZeroCopyOffset::InitOutputDataInfo(const vector &input_size_list op_name_ = op_desc->GetName(); (void)ge::AttrUtils::GetListInt(op_desc, ATTR_ZERO_COPY_BASIC_OFFSET, zero_copy_basic_offset_); (void)ge::AttrUtils::GetListInt(op_desc, ATTR_ZERO_COPY_RELATIVE_OFFSET, zero_copy_relative_offset_); - GE_CHK_BOOL_EXEC(zero_copy_basic_offset_.size() == zero_copy_relative_offset_.size(), return PARAM_INVALID, - "basic_offset_size should be equal to relative_offset_size"); + GE_CHK_BOOL_EXEC(zero_copy_basic_offset_.size() == zero_copy_relative_offset_.size(), + REPORT_INNER_ERROR("E19999", "basic_offset_size:%zu not equal to relative_offset_size:%zu, " + "check invalid", + zero_copy_basic_offset_.size(), zero_copy_relative_offset_.size()); + return PARAM_INVALID, + "[Check][Param] basic_offset_size:%zu should be equal to relative_offset_size:%zu", + zero_copy_basic_offset_.size(), zero_copy_relative_offset_.size()); int64_t virtual_addr_offset = op_desc->GetInputOffset().at(idx); IsL2Fusion(zero_copy_basic_offset_, virtual_addr_offset, fusion_flag); @@ -194,7 +205,8 @@ void ZeroCopyOffset::SetOutsideAddrsValue(ZeroCopyTask &zero_copy_task, void *ou for (uint32_t out_count = 0; out_count < GetAddrCount(); ++out_count) { auto args_addrs = outside_addrs_[out_count].find(outside_addr); if (args_addrs != outside_addrs_[out_count].end()) { - GE_CHK_STATUS(zero_copy_task.SetTaskArgsOffset(addr_val, offset), "Input args invalid."); + GE_CHK_STATUS(zero_copy_task.SetTaskArgsOffset(addr_val, offset), + "[Set][TaskArgsOffset] failed, Input args invalid, offset:%zu.", offset); void *args_val = static_cast(args) + offset; args_addrs->second.push_back(args_val); GELOGD("[ZCPY] set copy input: virtual_addr: 0x%lx, task_addr: %p, args: %p, offset: %zu.", addr_val, args_val, diff --git a/ge/graph/load/model_manager/zero_copy_task.cc b/ge/graph/load/model_manager/zero_copy_task.cc index c96dd8b7..4957f8ea 100755 --- a/ge/graph/load/model_manager/zero_copy_task.cc +++ b/ge/graph/load/model_manager/zero_copy_task.cc @@ -36,9 +36,9 @@ ZeroCopyTask::~ZeroCopyTask() { args_addr_ = nullptr; } */ Status ZeroCopyTask::SetTaskArgsOffset(uintptr_t addr, size_t offset) { if (offset + sizeof(uintptr_t) > args_size_) { - REPORT_INNER_ERROR("E19999", "Param offset:%zu + 8 > args_size_:%zu, check invalid", - offset, args_size_); - GELOGE(FAILED, "[ZCPY] %s set task args failed, args size: %zu, offset: %zu", name_.c_str(), args_size_, offset); + REPORT_INNER_ERROR("E19999", "Param offset:%zu + 8 > args_size_:%zu, check invalid", offset, args_size_); + GELOGE(FAILED, "[Check][Param] [ZCPY] %s set task args failed, args size:%zu, offset:%zu", + name_.c_str(), args_size_, offset); return FAILED; // unexpected error, need fix. } @@ -118,9 +118,8 @@ Status ZeroCopyTask::DistributeParam(bool async_mode, rtStream_t stream) { } if (rt_err != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMemcpyAsync or rtMemcpy failed, size:%zu, ret: 0x%X", - args_size_, rt_err); - GELOGE(RT_FAILED, "[ZCPY] %s distribute task param failed, error=0x%x", name_.c_str(), rt_err); + REPORT_CALL_ERROR("E19999", "Call rtMemcpyAsync or rtMemcpy failed, size:%zu, ret:0x%X", args_size_, rt_err); + GELOGE(RT_FAILED, "[Distribute][TaskParam] for %s failed, error = 0x%x", name_.c_str(), rt_err); return RT_ERROR_TO_GE_STATUS(rt_err); } diff --git a/ge/graph/manager/graph_caching_allocator.cc b/ge/graph/manager/graph_caching_allocator.cc index 75aa5c01..82bfbda9 100644 --- a/ge/graph/manager/graph_caching_allocator.cc +++ b/ge/graph/manager/graph_caching_allocator.cc @@ -112,7 +112,7 @@ Status CachingAllocator::Initialize(uint32_t device_id) { auto bin_ptr = new (std::nothrow) BlockBin(BlockComparator); if (bin_ptr == nullptr) { REPORT_CALL_ERROR("E19999", "New BlockBin fail, device_id:%u", device_id); - GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "Alloc BlockBin failed."); + GELOGE(ACL_ERROR_GE_MEMORY_ALLOCATION, "[Alloc][BlockBin] failed, device_id:%u", device_id); return ACL_ERROR_GE_MEMORY_ALLOCATION; } free_block_bins_[i] = bin_ptr; @@ -137,6 +137,7 @@ uint8_t *CachingAllocator::Malloc(size_t size, uint8_t *org_ptr, uint32_t device uint8_t *ptr = nullptr; Block *block = FindFreeBlock(size, org_ptr, device_id); if (block == nullptr) { + std::lock_guard lock(mutex_); if (ge::SUCCESS == TryExtendCache(size, device_id)) { block = FindFreeBlock(size, org_ptr, device_id); if (block != nullptr) { @@ -147,9 +148,8 @@ uint8_t *CachingAllocator::Malloc(size_t size, uint8_t *org_ptr, uint32_t device ptr = block->ptr; } if (ptr == nullptr) { - REPORT_INNER_ERROR("E19999", "FindFreeBlock fail, size:%zu, device_id:%u", - size, device_id); - GELOGE(FAILED, "Malloc failed device id = %u, size= %zu", device_id, size); + REPORT_INNER_ERROR("E19999", "FindFreeBlock fail, size:%zu, device_id:%u", size, device_id); + GELOGE(FAILED, "[Check][Param] FindFreeBlock failed device id = %u, size= %zu", device_id, size); } return ptr; } @@ -157,18 +157,16 @@ uint8_t *CachingAllocator::Malloc(size_t size, uint8_t *org_ptr, uint32_t device Status CachingAllocator::Free(uint8_t *ptr, uint32_t device_id) { GELOGI("Free device id = %u", device_id); if (ptr == nullptr) { - REPORT_INNER_ERROR("E19999", "Param ptr is nullptr, device_id:%u, check invalid", - device_id); - GELOGE(PARAM_INVALID, "Invalid memory pointer"); + REPORT_INNER_ERROR("E19999", "Param ptr is nullptr, device_id:%u, check invalid", device_id); + GELOGE(PARAM_INVALID, "[Check][Param] Invalid memory pointer, device_id:%u", device_id); return ge::PARAM_INVALID; } std::lock_guard lock(mutex_); auto it = allocated_blocks_.find(ptr); if (it == allocated_blocks_.end()) { - REPORT_INNER_ERROR("E19999", "Param ptr not allocated before, device_id:%u, check invalid", - device_id); - GELOGE(PARAM_INVALID, "Invalid memory pointer: %p", ptr); + REPORT_INNER_ERROR("E19999", "Param ptr not allocated before, device_id:%u, check invalid", device_id); + GELOGE(PARAM_INVALID, "[Check][Param] Param ptr not allocated before, device_id:%u", device_id); return ge::PARAM_INVALID; } Block *block = it->second; @@ -225,9 +223,8 @@ Block *CachingAllocator::FindFreeBlock(size_t size, uint8_t *org_ptr, uint32_t d Block key(device_id, size, org_ptr); BlockBin *bin = GetBlockBin(size); if (bin == nullptr) { - REPORT_INNER_ERROR("E19999", "GetBlockBin fail, size:%zu, device_id:%u", - size, device_id); - GELOGE(ge::FAILED, "Get block bin failed size = %zu", size); + REPORT_INNER_ERROR("E19999", "GetBlockBin fail, size:%zu, device_id:%u", size, device_id); + GELOGE(ge::FAILED, "[Get][BlockBin] failed, size:%zu, device_id:%u", size, device_id); return nullptr; } std::lock_guard lock(mutex_); @@ -258,9 +255,8 @@ Block *CachingAllocator::SplitBlock(Block *block, size_t size, BlockBin &bin, ui Block *remaining = block; Block *new_block = new (std::nothrow) Block(device_id, size, &bin, block->ptr); if (new_block == nullptr) { - REPORT_CALL_ERROR("E19999", "New Block fail, size:%zu, device_id:%u", - size, device_id); - GELOGE(ge::FAILED, "Alloc block failed size = %zu", size); + REPORT_CALL_ERROR("E19999", "New Block fail, size:%zu, device_id:%u", size, device_id); + GELOGE(ge::FAILED, "[Alloc][Block] failed, size:%zu, device_id:%u", size, device_id); return block; } new_block->prev = remaining->prev; @@ -285,7 +281,7 @@ Status CachingAllocator::TryExtendCache(size_t size, uint32_t device_id) { size_t free_cached_memory_size = FreeCachedBlocks(); memory_addr = memory_allocator_->MallocMemory(purpose, memory_size, device_id); if (memory_addr == nullptr) { - GELOGE(ge::FAILED, "TryExtendCache failed, no enough memory for size = %zu, device_id = %u", memory_size, + GELOGE(ge::FAILED, "[Malloc][Memory] failed, no enough memory for size = %zu, device_id = %u", memory_size, device_id); return ge::FAILED; } @@ -304,16 +300,14 @@ Status CachingAllocator::TryExtendCache(size_t size, uint32_t device_id) { Status CachingAllocator::AddToBlockBin(uint8_t *ptr, size_t size, uint32_t device_id) { BlockBin *bin = GetBlockBin(size); if (bin == nullptr) { - REPORT_INNER_ERROR("E19999", "GetBlockBin fail, size:%zu, device_id:%u", - size, device_id); - GELOGE(ge::FAILED, "Get block bin failed size = %zu", size); + REPORT_INNER_ERROR("E19999", "GetBlockBin fail, size:%zu, device_id:%u", size, device_id); + GELOGE(ge::FAILED, "[Get][BlockBin] failed, size:%zu, device_id:%u", size, device_id); return ge::FAILED; } Block *block = new (std::nothrow) Block(device_id, size, bin, nullptr); if (block == nullptr) { - REPORT_CALL_ERROR("E19999", "New Block fail, size:%zu, device_id:%u", - size, device_id); - GELOGE(ge::FAILED, "Alloc block failed size = %zu", size); + REPORT_CALL_ERROR("E19999", "New Block fail, size:%zu, device_id:%u", size, device_id); + GELOGE(ge::FAILED, "[Alloc][Block] failed, size:%zu, device_id:%u", size, device_id); return ge::FAILED; } diff --git a/ge/graph/manager/graph_context.cc b/ge/graph/manager/graph_context.cc index 3a705ad9..6d202cef 100644 --- a/ge/graph/manager/graph_context.cc +++ b/ge/graph/manager/graph_context.cc @@ -33,7 +33,7 @@ GraphContext::GraphContext(const GraphNodePtr &graph_node) { if (compute_graph_ == nullptr) { std::shared_ptr graph = graph_node->GetGraph(); if (graph == nullptr) { - GELOGE(GE_GRAPH_OPTIMIZE_COMPUTE_GRAPH_NULL, "compute_graph by graphNode is NULL!"); + GELOGE(GE_GRAPH_OPTIMIZE_COMPUTE_GRAPH_NULL, "[Get][Graph] failed, compute_graph by graphNode is NULL!"); return; } @@ -45,7 +45,7 @@ GraphContext::GraphContext(const GraphNodePtr &graph_node) { Status GraphContext::SetComputeGraph(const GraphNodePtr &graph_node) { if (graph_node == nullptr) { REPORT_INNER_ERROR("E19999", "Param graph_node is nullptr, check invalid"); - GELOGE(GE_GRAPH_PARAM_NULLPTR, "graphNode is NULL!"); + GELOGE(GE_GRAPH_PARAM_NULLPTR, "[Check][Param] graphNode is NULL!"); return GE_GRAPH_PARAM_NULLPTR; } @@ -56,7 +56,7 @@ Status GraphContext::SetComputeGraph(const GraphNodePtr &graph_node) { std::shared_ptr graph = graph_node->GetGraph(); if (graph == nullptr) { REPORT_INNER_ERROR("E19999", "Param graph in graph_node is nullptr, check invalid"); - GELOGE(GE_GRAPH_OPTIMIZE_COMPUTE_GRAPH_NULL, "compute_graph by graphNode is NULL!"); + GELOGE(GE_GRAPH_OPTIMIZE_COMPUTE_GRAPH_NULL, "[Get][Graph] failed, compute_graph by graphNode is NULL!"); return GE_GRAPH_OPTIMIZE_COMPUTE_GRAPH_NULL; } @@ -73,14 +73,15 @@ Status GraphContext::Finalize() const { return SUCCESS; } Status GraphContext::GetVariableTensor(const std::string &var_data_name, GeTensor &returned_tensor) { if (var_data_name.empty()) { REPORT_INNER_ERROR("E19999", "Param var_data_name is empty, check invalid"); - GELOGE(GE_GRAPH_EMPTY_STRING_NAME, "Variable data name is empty!"); + GELOGE(GE_GRAPH_EMPTY_STRING_NAME, "[Check][Param] Variable data name is empty!"); return GE_GRAPH_EMPTY_STRING_NAME; } if (GetVarNodeTensorTable().empty()) { REPORT_INNER_ERROR("E19999", "VarNodeTensorTable is empty, var_data_name:%s, check invalid", var_data_name.c_str()); - GELOGE(GE_GRAPH_EMPTY_VARIABLE_TENSOR_TABLE, "VarNodeTensorTable is empty!"); + GELOGE(GE_GRAPH_EMPTY_VARIABLE_TENSOR_TABLE, "[Check][Param] VarNodeTensorTable is empty, var_data_name:%s", + var_data_name.c_str()); return GE_GRAPH_EMPTY_VARIABLE_TENSOR_TABLE; } for (auto &var_record : GetVarNodeTensorTable()) { @@ -88,9 +89,8 @@ Status GraphContext::GetVariableTensor(const std::string &var_data_name, GeTenso returned_tensor.SetTensorDesc(var_record.second.GetTensorDesc()); auto ret = returned_tensor.SetData(var_record.second.GetData()); if (ret != SUCCESS) { - REPORT_INNER_ERROR("E19999", "SetData to tensor fail, var_data_name:%s", - var_data_name.c_str()); - GELOGE(ret, "Set Tensor data failed!"); + REPORT_INNER_ERROR("E19999", "SetData to tensor fail, var_data_name:%s", var_data_name.c_str()); + GELOGE(ret, "[Set][Data] to Tensor failed, var_data_name:%s", var_data_name.c_str()); return ret; } @@ -100,7 +100,8 @@ Status GraphContext::GetVariableTensor(const std::string &var_data_name, GeTenso REPORT_INNER_ERROR("E19999", "VarRecord with data_name:%s does not exist, check invalid", var_data_name.c_str()); - GELOGE(GE_GRAPH_VARIABLE_DOES_NOT_EXIST, "VarRecord with data_name %s does NOT exist!", var_data_name.c_str()); + GELOGE(GE_GRAPH_VARIABLE_DOES_NOT_EXIST, "[Check][Param] VarRecord with data_name %s does NOT exist!", + var_data_name.c_str()); return GE_GRAPH_VARIABLE_DOES_NOT_EXIST; } diff --git a/ge/graph/manager/graph_manager.cc b/ge/graph/manager/graph_manager.cc index 465ae749..273d2195 100755 --- a/ge/graph/manager/graph_manager.cc +++ b/ge/graph/manager/graph_manager.cc @@ -65,7 +65,6 @@ #include "graph/passes/merge_pass.h" #include "graph/passes/merge_input_memcpy_pass.h" #include "graph/passes/merge_to_stream_merge_pass.h" -#include "graph/passes/mark_force_unknown_for_cond_pass.h" #include "graph/passes/multi_batch_pass.h" #include "graph/passes/next_iteration_pass.h" #include "graph/passes/permute_pass.h" @@ -150,7 +149,8 @@ ge::Status CheckFpCeilingMode() { if (ret == ge::GRAPH_SUCCESS) { if (kValidFpCeilingMode.count(mode) == 0) { REPORT_INNER_ERROR("E19999", "Option ge.fpCeilingMode is invalid, value:%s", mode.c_str()); - GELOGE(ge::GE_GRAPH_OPTIONS_INVALID, "The fp_ceiling_mode %s is invalid, options are 0, 1, and 2.", mode.c_str()); + GELOGE(ge::GE_GRAPH_OPTIONS_INVALID, "[Get][Option] The fp_ceiling_mode %s is invalid, options are 0, 1, and 2.", + mode.c_str()); return ge::GE_GRAPH_OPTIONS_INVALID; } GELOGI("The parameter fp_ceiling_mode is set to %s.", mode.c_str()); @@ -179,33 +179,33 @@ Status GraphManager::Initialize(const std::map &options) { graph_run_listener_ = MakeShared(sync_run_mutex_, condition_); if (graph_run_listener_ == nullptr) { REPORT_CALL_ERROR("E19999", "New GraphModelListener fail"); - GELOGE(MEMALLOC_FAILED, "Make shared failed"); + GELOGE(MEMALLOC_FAILED, "[New][GraphModelListener] failed"); return MEMALLOC_FAILED; } // graph context graph_context_ = MakeShared(); if (graph_context_ == nullptr) { - REPORT_CALL_ERROR("E19999", "New GraphModelListener fail"); - GELOGE(MEMALLOC_FAILED, "Make shared failed."); + REPORT_CALL_ERROR("E19999", "New GraphContext fail"); + GELOGE(MEMALLOC_FAILED, "[New][GraphContext] failed."); return MEMALLOC_FAILED; } // parse option parameters Status ret = ParseOptions(options); if (ret != SUCCESS) { - GELOGE(ret, "[Initialize] parse options failed."); + GELOGE(ret, "[Parse][Options] failed."); return ret; } ret = CheckFpCeilingMode(); if (ret != SUCCESS) { - GELOGE(ret, "[Initialize] Check fp-ceiling-mode options failed."); + GELOGE(ret, "[Check][FpCeilingMode] failed."); return ret; } ret = graph_context_->Initialize(options); if (ret != SUCCESS) { - GELOGE(ret, "[Initialize] GraphContext initialize failed."); + GELOGE(ret, "[Initialize][GraphContext] failed."); return ret; } @@ -303,7 +303,7 @@ Status GraphManager::Finalize() { if (graph_context_ != nullptr) { Status ret_final = graph_context_->Finalize(); if (ret_final != SUCCESS) { - GELOGE(ret_final, "[GraphManager] graph context Finalize failed!"); + GELOGE(ret_final, "[Finalize][GraphContext] failed!"); unload_model_ret = ret_final; } } @@ -322,9 +322,8 @@ Status GraphManager::InitDynamicParams(ComputeGraphPtr &compute_graph) { std::string op_type; auto ret = GetOriginalType(node, op_type); if (ret != SUCCESS) { - REPORT_CALL_ERROR("E19999", "GetOriginalType from op:%s fail", - node->GetName().c_str()); - GELOGE(FAILED, "Failed to get node %s original type.", node->GetName().c_str()); + REPORT_CALL_ERROR("E19999", "GetOriginalType from op:%s fail", node->GetName().c_str()); + GELOGE(FAILED, "[Get][OriginalType] from op:%s failed.", node->GetName().c_str()); return FAILED; } if ((op_desc->GetType() == DATA) || (op_type == kGetNextName)) { @@ -336,7 +335,7 @@ Status GraphManager::InitDynamicParams(ComputeGraphPtr &compute_graph) { if (!options_.input_shape.empty() && !options_.dynamic_dims.empty()) { if (!ge::ParseInputShape(options_.input_shape, GetLocalOmgContext().input_dims, GetLocalOmgContext().user_input_dims, true)) { - GELOGE(GRAPH_PARAM_INVALID, "Failed to parse input shape: %s.", options_.input_shape.c_str()); + GELOGE(GRAPH_PARAM_INVALID, "[Parse][InputShape] %s failed.", options_.input_shape.c_str()); return GRAPH_PARAM_INVALID; } GetLocalOmgContext().dynamic_dims = options_.dynamic_dims; @@ -381,7 +380,7 @@ void GraphManager::RemoveAddGraphCondition(GraphId graph_id) { Status GraphManager::CheckRepeatAdd(uint32_t graph_id, bool &is_added) { uint32_t count = 0; if (GetGraphCount(graph_id, count) != SUCCESS) { - GELOGE(INTERNAL_ERROR, "Get graph [id:%u] count failed, graph might have not been added.", graph_id); + GELOGE(INTERNAL_ERROR, "[Get][GraphCount] failed, graph[id:%u] might have not been added.", graph_id); return INTERNAL_ERROR; } // previous thread owns same graph_id has been in the middle of the AddGraph procession @@ -394,7 +393,7 @@ Status GraphManager::CheckRepeatAdd(uint32_t graph_id, bool &is_added) { GraphNodePtr graph_node; Status ret = GetGraphNode(graph_id, graph_node); if (ret != SUCCESS) { - GELOGE(ret, "[AddGraph] GetGraphNode failed, graph_id = %u.", graph_id); + GELOGE(ret, "[Get][GraphNode] failed, graph_id = %u.", graph_id); return ret; } is_added = true; @@ -419,7 +418,7 @@ void GraphManager::SetSessionGraphId(ComputeGraphPtr compute_graph, uint32_t gra Status GraphManager::NotifyWaittingGraph(uint32_t graph_id) { uint32_t count = 0; if (GetGraphCount(graph_id, count) != SUCCESS) { - GELOGE(INTERNAL_ERROR, "Get graph [id:%u] count failed, graph might have not been added.", graph_id); + GELOGE(INTERNAL_ERROR, "[Get][GraphCount] failed, graph[id:%u] might have not been added.", graph_id); return INTERNAL_ERROR; } GELOGD("Add graph finished, graph_id:%u", graph_id); @@ -434,15 +433,13 @@ Status GraphManager::CreateGraphNode(uint32_t graph_id, const Graph &graph, const std::map &options) { GraphNodePtr graph_node = MakeShared(graph_id); GE_IF_BOOL_EXEC(graph_node == nullptr, - REPORT_CALL_ERROR("E19999", "New GraphNode fail, graph_id:%u", - graph_id); - GELOGE(FAILED, "GraphNode make shared failed"); + REPORT_CALL_ERROR("E19999", "New GraphNode fail, graph_id:%u", graph_id); + GELOGE(FAILED, "[New][GraphNode] fail, graph_id:%u", graph_id); return FAILED); std::shared_ptr graph_ptr = MakeShared(graph); GE_IF_BOOL_EXEC(graph_ptr == nullptr, - REPORT_CALL_ERROR("E19999", "New Graph fail, graph_id:%u", - graph_id); - GELOGE(FAILED, "GraphPtr make shared failed"); + REPORT_CALL_ERROR("E19999", "New Graph fail, graph_id:%u", graph_id); + GELOGE(FAILED, "[New][Graph] fail, graph_id:%u", graph_id); return FAILED); // update option about tuning graph ParseOption(options, BUILD_MODE, options_.build_mode); @@ -460,7 +457,7 @@ Status GraphManager::SetStagesOptions(uint32_t graph_id, const GraphManagerOptio stages.preparer.SetOptions(options_); Status status = stages.optimizer.SetOptions(options_); if (status != SUCCESS) { - GELOGE(status, "Graph optimizer set options failed."); + GELOGE(status, "[Set][Options] for Graph optimizer failed, graph id:%u.", graph_id); return status; } stages.builder.SetOptions(options_); @@ -518,7 +515,8 @@ Status GraphManager::AddGraph(const GraphId &graph_id, const Graph &graph, if (GetAddGraphCondition(graph_id) == kDoneAdded) { GraphNodePtr graph_node; if (GetGraphNode(graph_id, graph_node) != SUCCESS) { - GELOGE(GE_GRAPH_GRAPH_NOT_EXIST, "Graph not exist while done adding previously, graph_id = %u.", graph_id); + GELOGE(GE_GRAPH_GRAPH_NOT_EXIST, "[Get][GraphNode] failed, Graph not exist while done adding previously, " + "graph_id = %u.", graph_id); return GE_GRAPH_GRAPH_NOT_EXIST; } graph_node->IncreaseLoadCount(); @@ -529,7 +527,7 @@ Status GraphManager::AddGraph(const GraphId &graph_id, const Graph &graph, // done adding graph of the former graph, avoiding repeatively adding same graph. bool is_added = false; if (CheckRepeatAdd(graph_id, is_added) != SUCCESS) { - GELOGE(INTERNAL_ERROR, "CheckRepeatAdd for graph[id:%u] failed.", graph_id); + GELOGE(INTERNAL_ERROR, "[Check][RepeatAdd] for graph[id:%u] failed.", graph_id); return INTERNAL_ERROR; } // The former graph (from different thread) owns same graph id has been successfully added. @@ -539,7 +537,7 @@ Status GraphManager::AddGraph(const GraphId &graph_id, const Graph &graph, // Do add graph SetAddGraphCondition(graph_id, kStartAdd); if (CheckGraphAdded(graph_id, graph) != SUCCESS) { - GELOGE(FAILED, "AddGraph failed."); + GELOGE(FAILED, "[Check][GraphAdded] failed, graph id:%u.", graph_id); return FAILED; } GE_CHK_STATUS_RET(ModifyDataIndex(graph, options)); @@ -549,7 +547,7 @@ Status GraphManager::AddGraph(const GraphId &graph_id, const Graph &graph, SetSessionGraphId(compute_graph, graph_id); if (CreateGraphNode(graph_id, graph, options) != SUCCESS) { - GELOGE(FAILED, "Failed to create graph_node."); + GELOGE(FAILED, "[Create][GraphNode] failed, graph id:%u.", graph_id); return FAILED; } @@ -558,12 +556,12 @@ Status GraphManager::AddGraph(const GraphId &graph_id, const Graph &graph, GetLocalOmgContext().output_type = options_.output_datatype; } if (InitDynamicParams(compute_graph) != SUCCESS) { - GELOGE(GRAPH_PARAM_INVALID, "Failed to init params when online infer is dynamic."); + GELOGE(GRAPH_PARAM_INVALID, "[Init][Params] failed, when online infer is dynamic, graph id:%u.", graph_id); return GRAPH_PARAM_INVALID; } if (SetStagesOptions(graph_id, options_) != SUCCESS) { - GELOGE(INTERNAL_ERROR, "Set stage options failed."); + GELOGE(INTERNAL_ERROR, "[Set][StagesOptions] failed, graph id:%u.", graph_id); return INTERNAL_ERROR; } @@ -571,7 +569,7 @@ Status GraphManager::AddGraph(const GraphId &graph_id, const Graph &graph, SetAddGraphCondition(graph_id, kDoneAdded); // There are threads waitting for adding same graph if (NotifyWaittingGraph(graph_id) != SUCCESS) { - GELOGE(INTERNAL_ERROR, "NotifyWaittingGraph failed."); + GELOGE(INTERNAL_ERROR, "[Notify][WaittingGraph] failed, graph id:%u.", graph_id); return INTERNAL_ERROR; } return SUCCESS; @@ -586,14 +584,13 @@ Status GraphManager::CheckGraphAdded(const GraphId &graph_id, const Graph &graph && graph_has_been_added) { REPORT_INNER_ERROR("E19999", "Get Attr:%s from graph:%u fail.", ATTR_NAME_GRAPH_HAS_BEEN_ADDED.c_str(), graph_id); - GELOGE(GE_GRAPH_GRAPH_ALREADY_EXIST, - "[GraphManager] same graph object can not be added again, graph_id = %u.", graph_id); + GELOGE(GE_GRAPH_GRAPH_ALREADY_EXIST, "[Get][Attr] %s from graph:%u fail.", + ATTR_NAME_GRAPH_HAS_BEEN_ADDED.c_str(), graph_id); return GE_GRAPH_GRAPH_ALREADY_EXIST; } } else { - REPORT_INNER_ERROR("E19999", "compute_graph from graph:%u is nullptr, check invalid", - graph_id); - GELOGE(FAILED, "compute graph is null"); + REPORT_INNER_ERROR("E19999", "compute_graph from graph:%u is nullptr, check invalid", graph_id); + GELOGE(FAILED, "[Get][ComputeGraph] failed, compute graph from graph:%u is nullptr", graph_id); return FAILED; } return SUCCESS; @@ -603,11 +600,11 @@ Status GraphManager::AddGraphWithCopy(const GraphId &graph_id, const Graph &grap const std::map &options, const OmgContext &omg_context) { if (HasGraphNode(graph_id)) { - GELOGE(GE_GRAPH_GRAPH_ALREADY_EXIST, "[GraphManager] graph exists, graph_id = %u", graph_id); + GELOGE(GE_GRAPH_GRAPH_ALREADY_EXIST, "[Has][GraphNode] graph exists, graph_id = %u", graph_id); return GE_GRAPH_GRAPH_ALREADY_EXIST; } if (CheckGraphAdded(graph_id, graph) != SUCCESS) { - GELOGE(FAILED, "AddGraphWithCopy failed."); + GELOGE(FAILED, "[Check][GraphAdded] failed, graph_id = %u", graph_id); return FAILED; } IncreaseGraphCount(graph_id); @@ -621,7 +618,7 @@ Status GraphManager::AddGraphWithCopy(const GraphId &graph_id, const Graph &grap SetSessionGraphId(new_compute_graph, graph_id); std::shared_ptr new_graph_ptr = GraphUtils::CreateGraphPtrFromComputeGraph(new_compute_graph); if (CreateGraphNode(graph_id, *new_graph_ptr, options) != SUCCESS) { - GELOGE(FAILED, "Failed to create graph_node."); + GELOGE(FAILED, "[Create][GraphNode] failed, graph_id = %u", graph_id); return FAILED; } @@ -630,12 +627,12 @@ Status GraphManager::AddGraphWithCopy(const GraphId &graph_id, const Graph &grap GetLocalOmgContext().output_type = options_.output_datatype; } if (InitDynamicParams(new_compute_graph) != SUCCESS) { - GELOGE(GRAPH_PARAM_INVALID, "Failed to init params when online infer is dynamic."); + GELOGE(GRAPH_PARAM_INVALID, "[Init][Params] failed, when online infer is dynamic, graph_id = %u", graph_id); return GRAPH_PARAM_INVALID; } if (SetStagesOptions(graph_id, options_) != SUCCESS) { - GELOGE(INTERNAL_ERROR, "Set stage options failed."); + GELOGE(INTERNAL_ERROR, "[Set][StagesOptions] failed, graph_id = %u", graph_id); return INTERNAL_ERROR; } @@ -656,9 +653,9 @@ Status GraphManager::MergeSubGraph(ComputeGraphPtr &compute_graph, const ge::Com Status ret_topo = compute_graph->TopologicalSorting(); if (ret_topo != SUCCESS) { - REPORT_CALL_ERROR("E19999", "TopologicalSorting fail, graph_id:%u", - compute_graph->GetGraphID()); - GELOGE(ret_topo, "[GraphManager]: TopologicalSorting the merged graph failed."); + REPORT_CALL_ERROR("E19999", "TopologicalSorting fail, graph_id:%u", compute_graph->GetGraphID()); + GELOGE(ret_topo, "[Call][TopologicalSorting] for the merged graph failed, graph_id:%u", + compute_graph->GetGraphID()); return ret_topo; } } else { @@ -693,16 +690,16 @@ Status GraphManager::CopySubGraphAndMarkFusion(const ComputeGraphPtr &compute_gr std::vector output_nodes; ComputeGraphPtr new_compute_graph = GraphUtils::CloneGraph(old_compute_graph, "", input_nodes, output_nodes); if (new_compute_graph == nullptr) { - REPORT_CALL_ERROR("E19999", "CloneGraph fail, graph_id:%u", - compute_graph->GetGraphID()); - GELOGE(INTERNAL_ERROR, "Clone graph failed."); + REPORT_CALL_ERROR("E19999", "CloneGraph fail, graph_id:%u", compute_graph->GetGraphID()); + GELOGE(INTERNAL_ERROR, "[Clone][Graph] failed, graph_id:%u", compute_graph->GetGraphID()); return INTERNAL_ERROR; } copy_graphs.emplace(old_compute_graph->GetName(), new_compute_graph); if (!AttrUtils::SetBool(old_compute_graph, ATTR_NAME_NEED_LX_FUSION, true)) { REPORT_INNER_ERROR("E19999", "Set Attr:%s to graph:%u fail", ATTR_NAME_NEED_LX_FUSION.c_str(), old_compute_graph->GetGraphID()); - GELOGE(INTERNAL_ERROR, "Set attr lx_fusion to graph failed."); + GELOGE(INTERNAL_ERROR, "[Set][Attr] %s to graph:%u failed.", + ATTR_NAME_NEED_LX_FUSION.c_str(), old_compute_graph->GetGraphID()); return INTERNAL_ERROR; } } @@ -740,7 +737,7 @@ Status GraphManager::OptimizeSubGraphWithMultiThreads(ComputeGraphPtr compute_gr ErrorManager::GetInstance().GetErrorManagerContext(), GetThreadLocalContext()); if (!f.valid()) { - GELOGE(FAILED, "Future is invalid"); + GELOGE(FAILED, "[Call][Commit] failed, Future is invalid, session_id:%lu", session_id); return FAILED; } vector_future.emplace_back(std::move(f)); @@ -757,7 +754,7 @@ Status GraphManager::OptimizeSubGraphWithMultiThreads(ComputeGraphPtr compute_gr ErrorManager::GetInstance().GetErrorManagerContext(), GetThreadLocalContext()); if (!f.valid()) { - GELOGE(FAILED, "Future is invalid"); + GELOGE(FAILED, "[Call][Commit] failed, Future is invalid, session_id:%lu", session_id); return FAILED; } vector_future.emplace_back(std::move(f)); @@ -768,7 +765,7 @@ Status GraphManager::OptimizeSubGraphWithMultiThreads(ComputeGraphPtr compute_gr Status ret_status = vector_future[i].get(); if (ret_status != SUCCESS) { REPORT_CALL_ERROR("E19999", "subgraph %zu optimize failed", i); - GELOGE(ret_status, "subgraph %zu optimize failed", i); + GELOGE(ret_status, "[Check][Param] subgraph %zu optimize failed", i); return ret_status; } } @@ -779,7 +776,7 @@ bool GraphManager::CheckAllFusionOptimizeSuccess(const ComputeGraphPtr &compute_ Graph2SubGraphInfoList &sub_graph_map) { if (compute_graph == nullptr) { REPORT_INNER_ERROR("E19999", "Param compute_graph is nullptr, check invalid"); - GELOGE(PARAM_INVALID, "Input param compute_graph is nullptr."); + GELOGE(PARAM_INVALID, "[Check][Param] Input param compute_graph is nullptr."); return false; } @@ -820,7 +817,8 @@ Status GraphManager::ReplaceSubgraphWithOriGraph(const ComputeGraphPtr &compute_ if (iter == copy_graphs.end()) { REPORT_INNER_ERROR("E19999", "Can not find subgraph:%s in copy graphs, check invalid", subgraph->GetSubGraph()->GetName().c_str()); - GELOGE(FAILED, "Can not find subgraph:%s in copy graphs.", subgraph->GetSubGraph()->GetName().c_str()); + GELOGE(FAILED, "[Check][Param] Can not find subgraph:%s in copy graphs.", + subgraph->GetSubGraph()->GetName().c_str()); return FAILED; } subgraph->SetSubGraph(iter->second); @@ -833,7 +831,8 @@ Status GraphManager::ReplaceSubgraphWithOriGraph(const ComputeGraphPtr &compute_ if (iter == copy_graphs.end()) { REPORT_INNER_ERROR("E19999", "Can not find subgraph:%s in copy graphs, check invalid", subgraph->GetSubGraph()->GetName().c_str()); - GELOGE(FAILED, "Can not find subgraph:%s in copy graphs.", subgraph->GetSubGraph()->GetName().c_str()); + GELOGE(FAILED, "[Check][Param] Can not find subgraph:%s in copy graphs.", + subgraph->GetSubGraph()->GetName().c_str()); return FAILED; } subgraph->SetSubGraph(iter->second); @@ -851,7 +850,7 @@ Status GraphManager::SetSubgraph(uint64_t session_id, ComputeGraphPtr compute_gr options_.build_step.c_str()); Status ret = OptimizeSubGraphWithMultiThreads(compute_graph, sub_graph_map, session_id); if (ret != SUCCESS) { - GELOGE(ret, "Multiply optimize subgraph failed"); + GELOGE(ret, "[Call][OptimizeSubGraphWithMultiThreads] failed, ret:%d, session_id:%lu", ret, session_id); return ret; } return SUCCESS; @@ -887,13 +886,15 @@ Status GraphManager::PreRunOptimizeOriginalGraph(const GraphNodePtr &graph_node, GM_RUN_AND_DUMP_PERF("OptimizeSwitchOp", stages.preparer.SwitchOpOptimize, compute_graph); } GM_RUN_AND_DUMP_PERF("Optimize1", OptimizeStage1, compute_graph); + GM_RUN_AND_DUMP_PERF("OptimizeAfterStage1", stages.optimizer.OptimizeAfterStage1, compute_graph); GM_RUN_AND_DUMP_PERF("InferShape2", compute_graph->InferShapeInNeed); PassManager graph_pass; GE_CHK_STATUS_RET(graph_pass.AddPass("PreRun::CtrlEdgeTransferPass", new (std::nothrow) CtrlEdgeTransferPass)) GE_CHK_STATUS_RET(graph_pass.Run(compute_graph)); - GE_CHK_STATUS_RET(stages.optimizer.IdentifyReference(compute_graph), "Identify reference failed."); + GE_CHK_STATUS_RET(stages.optimizer.IdentifyReference(compute_graph), + "[Identify][Reference] failed, graph:%s.", compute_graph->GetName().c_str()); GELOGD("PreRun:PreRunOptimizeOriginalGraph success."); return SUCCESS; } @@ -931,9 +932,8 @@ Status GraphManager::PreRunAfterOptimizeSubGraph(const GraphNodePtr &graph_node, Status ret = compute_graph->TopologicalSorting(); if (ret != SUCCESS) { - REPORT_CALL_ERROR("E19999", "TopologicalSorting fail, graph_id:%u", - compute_graph->GetGraphID()); - GELOGE(ret, "Graph topological sort failed, ret:%d.", ret); + REPORT_CALL_ERROR("E19999", "TopologicalSorting fail, graph_id:%u", compute_graph->GetGraphID()); + GELOGE(ret, "[Call][TopologicalSorting] fail, graph_id:%u", compute_graph->GetGraphID()); return ret; } @@ -950,14 +950,14 @@ Status GraphManager::SetRtContext(rtContext_t rt_context, rtCtxMode_t mode, uint if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtCtxCreate faileded, session_id:%lu, graph_id:%u, mode:%d", session_id, graph_id, mode); - GELOGE(FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + GELOGE(FAILED, "[Call][RtCtxCreate] faileded, session_id:%lu, graph_id:%u, mode:%d", session_id, graph_id, mode); return FAILED; } rt_ret = rtCtxSetCurrent(rt_context); if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtCtxSetCurrent failed, session_id:%lu, graph_id:%u, mode:%d", session_id, graph_id, mode); - GELOGE(FAILED, "Call rt api failed, ret: 0x%X", rt_ret); + GELOGE(FAILED, "[Call][RtCtxSetCurrent] failed, session_id:%lu, graph_id:%u, mode:%d", session_id, graph_id, mode); return FAILED; } RtContextUtil::GetInstance().AddRtContext(session_id, graph_id, rt_context); @@ -971,7 +971,7 @@ Status GraphManager::RunCustomPass(const GraphNodePtr &graph_node) { GE_TIMESTAMP_START(RunCustomPass); GraphPtr graph = std::const_pointer_cast(const_graph); - GE_CHK_STATUS_RET(CustomPassHelper::Instance().Run(graph), "Graph[%s] run custom pass fail.", + GE_CHK_STATUS_RET(CustomPassHelper::Instance().Run(graph), "[Call][Run] for Graph[%s] fail.", comp_graph->GetName().c_str()); GE_TIMESTAMP_END(RunCustomPass, "GraphBuilder::RunCustomPass"); return SUCCESS; @@ -987,7 +987,7 @@ Status GraphManager::PreRun(const GraphNodePtr &graph_node, const std::vectorSetSessionID(session_id); auto analyzer_instance = Analyzer::GetInstance(); GE_CHK_STATUS_RET(analyzer_instance->BuildJsonObject(session_id, compute_graph->GetGraphID()), - "BuildJsonObject Failed") + "[Build][JsonObject] Failed, session_id:%lu", session_id) GEEVENT("PreRun start: graph node size %zu, session id %lu, graph id %u, graph name %s.", compute_graph->GetDirectNodesSize(), session_id, compute_graph->GetGraphID(), @@ -996,7 +996,7 @@ Status GraphManager::PreRun(const GraphNodePtr &graph_node, const std::vectorGetGraphID()); if (ret != SUCCESS) { - GELOGE(ret, "Set rt context failed."); + GELOGE(ret, "[Set][RtContext] failed, session_id:%lu, graph_id:%u.", session_id, compute_graph->GetGraphID()); return ret; } @@ -1010,17 +1010,20 @@ Status GraphManager::PreRun(const GraphNodePtr &graph_node, const std::vectorGetName().c_str()); + GELOGE(ret, "[Run][PreRunOptimizeOriginalGraph] failed for graph:%s, session_id:%lu", + compute_graph->GetName().c_str(), session_id); return ret; } } ErrorManager::GetInstance().SetStage(error_message::kModelCompile, error_message::kSubGraphOptimize); // set fuzz compile flag after origin graph optimize - GE_CHK_STATUS_RET(SetFuzzCompileFlag(compute_graph), "Set fuzz compile flag failed."); + GE_CHK_STATUS_RET(SetFuzzCompileFlag(compute_graph), + "[Set][FuzzCompileFlag] failed for graph:%s.", compute_graph->GetName().c_str()); ret = PreRunOptimizeSubGraph(graph_node, compute_graph, session_id); if (ret != SUCCESS) { - GELOGE(ret, "Run PreRunOptimizeSubGraph failed for graph:%s.", compute_graph->GetName().c_str()); + GELOGE(ret, "[Run][PreRunOptimizeSubGraph] failed for graph:%s, session_id:%lu.", + compute_graph->GetName().c_str(), session_id); return ret; } @@ -1034,7 +1037,8 @@ Status GraphManager::PreRun(const GraphNodePtr &graph_node, const std::vectorGetName().c_str()); + GELOGE(ret, "[Run][PreRunAfterOptimizeSubGraph] failed for graph:%s, session_id:%lu.", + compute_graph->GetName().c_str(), session_id); return ret; } } @@ -1059,7 +1063,7 @@ Status GraphManager::SetFuzzCompileFlag(ComputeGraphPtr &compute_graph) { GE_CHECK_NOTNULL(op_desc); GELOGD("Fuzz compile flag is %d.", GetLocalOmgContext().fuzz_compile_flag); if (!AttrUtils::SetBool(op_desc, ATTR_NAME_FUZZ_BUILD, GetLocalOmgContext().fuzz_compile_flag)) { - GELOGE(FAILED, "[Set][ATTR_NAME_FUZZ_BUILD]Failed to set fuzz build attr to %s.", op_desc->GetName().c_str()); + GELOGE(FAILED, "[Set][ATTR] %s to %s failed.", ATTR_NAME_FUZZ_BUILD.c_str(), op_desc->GetName().c_str()); return FAILED; } } @@ -1075,7 +1079,7 @@ Status GraphManager::SubexpressionMigration(ComputeGraphPtr &compute_graph) { auto ret = pass_manager.Run(compute_graph); GE_TIMESTAMP_END(SubexpressionMigrationPass, "GraphManager::SubexpressionMigration"); if (ret != SUCCESS && ret != NOT_CHANGED) { - GELOGE(ret, "Run SubexpressionMigrationPass failed, ret:%u.", ret); + GELOGE(ret, "[Run][SubexpressionMigrationPass] failed, ret:%u.", ret); return ret; } @@ -1092,7 +1096,7 @@ Status GraphManager::StartForRunGraph(const GraphNodePtr &graph_node, const std: REPORT_INNER_ERROR("E19999", "Graph:%u has not build before, can't run directly, " "check invalid", graph_node->GetGraphId()); GELOGE(PARAM_INVALID, - "The graph %u need to re-build, you should remove it from GE " + "[Get][BuildFlag] The graph %u need to re-build, you should remove it from GE " "first, then AddGraph again and rebuild it.", graph_node->GetGraphId()); return PARAM_INVALID; @@ -1105,7 +1109,7 @@ Status GraphManager::StartForRunGraph(const GraphNodePtr &graph_node, const std: // release rts generate context RtContextUtil::GetInstance().DestroyRtContexts(session_id, graph_node->GetGraphId()); if (ret != SUCCESS) { - GELOGE(ret, "PreRun Failed, graph_id:%u.", graph_node->GetGraphId()); + GELOGE(ret, "[Call][PreRun] Failed, graph_id:%u, session_id:%lu.", graph_node->GetGraphId(), session_id); return ret; } } @@ -1116,7 +1120,7 @@ Status GraphManager::StartForRunGraph(const GraphNodePtr &graph_node, const std: ret = LoadGraphAsync(ge_root_model, graph_node); } if (ret != SUCCESS) { - GELOGE(ret, "LoadGraph Failed."); + GELOGE(ret, "[Load][Graph] Failed, graph_id:%u.", graph_node->GetGraphId()); return ret; } graph_node->SetBuildFlag(true); @@ -1130,7 +1134,7 @@ Status GraphManager::StartForRunGraph(const GraphNodePtr &graph_node, const std: ret = LoadGraphAsync(ge_root_model_ptr, graph_node); } if (ret != SUCCESS) { - GELOGE(ret, "LoadGraph Failed."); + GELOGE(ret, "[Load][Graph] Failed, graph_id:%u.", graph_node->GetGraphId()); return ret; } } @@ -1161,7 +1165,7 @@ Status GraphManager::LoadGraph(const GeRootModelPtr &ge_root_model, const GraphN Status ret = GraphLoader::LoadModelOnline(model_id_info.model_id, ge_root_model, model_listener); GE_TIMESTAMP_EVENT_END(LoadGraph, "GraphManager::LoadGraph"); if (ret != SUCCESS) { - GELOGE(ret, "[StartForRunGraph] LoadGraph Failed"); + GELOGE(ret, "[Load][Model] failed, ret:%d", ret); graph_node->SetRunFlag(false); return ret; } @@ -1261,7 +1265,7 @@ Status GraphManager::InnerRunGraph(GraphNodePtr &graph_node, const GraphId &grap const std::vector &inputs, std::vector &outputs) { Status ret = graph_executor_.SetCondition(&sync_run_mutex_, &condition_, graph_run_listener_); if (ret != SUCCESS) { - GELOGE(GE_GRAPH_RUNGRAPH_FAILED, "[RunGraph] set condition failed, graph_id = %u.", graph_id); + GELOGE(GE_GRAPH_RUNGRAPH_FAILED, "[Set][Condition] failed, graph_id = %u.", graph_id); graph_node->SetRunFlag(false); return GE_GRAPH_RUNGRAPH_FAILED; } @@ -1274,7 +1278,7 @@ Status GraphManager::InnerRunGraph(GraphNodePtr &graph_node, const GraphId &grap graph_node->SetRunFlag(false); if (ret != SUCCESS) { - GELOGE(ret, "[RunGraph] execute graph failed, graph_id = %u.", graph_id); + GELOGE(ret, "[Execute][Graph] failed, graph_id = %u.", graph_id); return ret; } return SUCCESS; @@ -1284,8 +1288,7 @@ Status GraphManager::InnerRunGraphWithStream(GraphNodePtr &graph_node, const Gra const std::vector &inputs, std::vector &outputs) { auto ret = graph_executor_.SetCondition(&sync_run_mutex_, &condition_, graph_run_listener_); if (ret != SUCCESS) { - GELOGE(GE_GRAPH_RUNGRAPH_FAILED, "[Run][GraphWithStreamAsync] set condition failed, " - "graph id = %u, stream = %p.", graph_id, stream); + GELOGE(GE_GRAPH_RUNGRAPH_FAILED, "[Set][Condition] failed, graph id = %u, stream = %p.", graph_id, stream); graph_node->SetRunFlag(false); return GE_GRAPH_RUNGRAPH_FAILED; } @@ -1294,7 +1297,7 @@ Status GraphManager::InnerRunGraphWithStream(GraphNodePtr &graph_node, const Gra graph_node->SetRunFlag(false); graph_node->SetIsSpecificStream(false); if (ret != SUCCESS) { - GELOGE(ret, "[Run][GraphWithStreamAsync] execute graph failed, graph id = %u, stream = %p.", graph_id, stream); + GELOGE(ret, "[Execute][Graph] With Stream failed, graph id = %u, stream = %p.", graph_id, stream); return ret; } GELOGI("[Run][GraphWithStreamAsync] run graph success, graph id = %u, stream = %p.", graph_id, stream); @@ -1316,18 +1319,20 @@ Status GraphManager::RunGraphWithStreamAsync(const GraphId &graph_id, rtStream_t Status ret = GetGraphNode(graph_id, graph_node); if (ret != SUCCESS) { REPORT_INNER_ERROR("E19999", "graph id = %u not exist in graph_map, check invalid.", graph_id); - GELOGE(ret, "Run graph with stream async graph not exist, graph id = %u.", graph_id); + GELOGE(ret, "[Get][GraphNode] failed, Run graph with stream async, graph not exist, graph id = %u.", graph_id); return ret; } if (graph_node == nullptr) { REPORT_INNER_ERROR("E19999", "Graph node is nullptr in graph_map, graph id = %u, check invalid.", graph_id); - GELOGE(GE_GRAPH_GRAPH_NODE_NULL, "Run graph with stream async graph node is NULL, graph id = %u.", graph_id); + GELOGE(GE_GRAPH_GRAPH_NODE_NULL, "[Check][Param] Run graph with stream async, graph node is NULL, " + "graph id = %u.", graph_id); return GE_GRAPH_GRAPH_NODE_NULL; } if (graph_node->GetRunFlag()) { REPORT_INNER_ERROR("E19999", "Graph is already running, can't be run again, graph id = %u, " "check invalid.", graph_id); - GELOGE(GE_GRAPH_ALREADY_RUNNING, "Run graph with stream async graph already running, graph id = %u.", graph_id); + GELOGE(GE_GRAPH_ALREADY_RUNNING, "[Get][RunFlag] Run graph with stream async graph already running, " + "graph id = %u.", graph_id); return GE_GRAPH_ALREADY_RUNNING; } @@ -1345,7 +1350,7 @@ Status GraphManager::RunGraphWithStreamAsync(const GraphId &graph_id, rtStream_t GeRootModelPtr ge_root_model = nullptr; ret = StartForRunGraph(graph_node, inputs, ge_root_model, session_id); if (ret != SUCCESS) { - GELOGE(ret, "[Run][GraphWithStreamAsync] StartForRunGraph failed!"); + GELOGE(ret, "[Call][StartForRunGraph] failed, session_id:%lu", session_id); graph_node->SetRunFlag(false); return ret; } @@ -1366,23 +1371,20 @@ Status GraphManager::RunGraph(const GraphId &graph_id, const std::vectorGetRunFlag()) { - REPORT_INNER_ERROR("E19999", "Graph is already running, can't be run again, graph_id:%u, " - "check invalid", graph_id); - GELOGE(GE_GRAPH_ALREADY_RUNNING, "[RunGraph] graph already running, graph id = %u", graph_id); + REPORT_INNER_ERROR("E19999", "Graph is already running, can't be run again, graph_id:%u, check invalid", graph_id); + GELOGE(GE_GRAPH_ALREADY_RUNNING, "[Get][RunFlag] graph already running, graph id = %u", graph_id); return GE_GRAPH_ALREADY_RUNNING; } @@ -1396,8 +1398,8 @@ Status GraphManager::RunGraph(const GraphId &graph_id, const std::vectorSetRunFlag(false); return ret; } @@ -1426,7 +1428,7 @@ Status GraphManager::RunGraph(const GraphId &graph_id, const std::vectorIsSummaryGraph()) { ret = SummaryHandle(graph_id, outputs); if (ret != SUCCESS) { - GELOGE(ret, "[RunGraph] SummaryHandle failed!"); + GELOGE(ret, "[Call][SummaryHandle] failed, graph_id:%u", graph_id); } } @@ -1437,7 +1439,7 @@ Status GraphManager::RunGraph(const GraphId &graph_id, const std::vectorInitFlag()) { - REPORT_INNER_ERROR("E19999", "GELib is not init before, graph_id:%u, check invalid", - graph_id); - GELOGE(GE_CLI_GE_NOT_INITIALIZED, "GE is not initialized"); + REPORT_INNER_ERROR("E19999", "GELib is not init before, graph_id:%u, check invalid", graph_id); + GELOGE(GE_CLI_GE_NOT_INITIALIZED, "[Get][GELib] GELib is not init before, graph_id:%u", graph_id); return GE_CLI_GE_NOT_INITIALIZED; } @@ -1525,7 +1524,9 @@ Status GraphManager::BuildGraphForUnregisteredOp(const GraphId &graph_id, const REPORT_INNER_ERROR("E19999", "GetOpsKernelInfoStore fail for op:%s(%s), kernel_lib_name:%s, graph_id:%u, " "check invalid", op_desc->GetName().c_str(), op_desc->GetType().c_str(), op_desc->GetOpKernelLibName().c_str(), graph_id); - GELOGE(FAILED, "Get op kernel info store failed"); + GELOGE(FAILED, "[Get][OpsKernelInfoStore] fail for op:%s(%s), kernel_lib_name:%s, graph_id:%u", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), + op_desc->GetOpKernelLibName().c_str(), graph_id); return FAILED; } @@ -1534,8 +1535,7 @@ Status GraphManager::BuildGraphForUnregisteredOp(const GraphId &graph_id, const REPORT_CALL_ERROR("E19999", "Call CompileOp fail for op:%s(%s), kernel_lib_name:%s, graph_id:%u, " "check invalid", op_desc->GetName().c_str(), op_desc->GetType().c_str(), op_desc->GetOpKernelLibName().c_str(), graph_id); - GELOGE(FAILED, "Get op kernel info store failed"); - GELOGE(ret, "Compile op failed, op = %s, graph_id = %u.", op_desc->GetName().c_str(), graph_id); + GELOGE(ret, "[Compile][Op] failed, op = %s, graph_id = %u.", op_desc->GetName().c_str(), graph_id); return ret; } } @@ -1558,23 +1558,21 @@ Status GraphManager::BuildGraph(const GraphId &graph_id, const std::vectorGetRunFlag()) { REPORT_INNER_ERROR("E19999", "Graph is already running, can't be run again, graph_id:%u, " "check invalid", graph_id); - GELOGE(GE_GRAPH_ALREADY_RUNNING, "[BuildGraph] graph already running, graph id = %u", graph_node->GetGraphId()); + GELOGE(GE_GRAPH_ALREADY_RUNNING, "[Get][RunFlag] graph already running, graph id = %u", graph_node->GetGraphId()); return GE_GRAPH_ALREADY_RUNNING; } @@ -1587,7 +1585,7 @@ Status GraphManager::BuildGraph(const GraphId &graph_id, const std::vectorSetRunFlag(false); if (ret != SUCCESS) { - GELOGE(GE_GRAPH_PRERUN_FAILED, "[BuildGraph] StartForRunGraph failed! graph_id:%u.", graph_id); + GELOGE(GE_GRAPH_PRERUN_FAILED, "[Call][StartForRunGraph] failed! graph_id:%u.", graph_id); return GE_GRAPH_PRERUN_FAILED; } @@ -1607,18 +1605,18 @@ Status GraphManager::BuildGraph(const GraphId &graph_id, const std::vector &attrs, const std::vector &inputs, const std::vector &outputs) { - GE_CHK_BOOL_EXEC(ge::AttrUtils::SetStr(&model, "ATTR_MODEL_OP_TYPE", type), return FAILED, "Set Op[%s] type fail", - type.c_str()); + GE_CHK_BOOL_EXEC(ge::AttrUtils::SetStr(&model, "ATTR_MODEL_OP_TYPE", type), return FAILED, + "[Set][Str] model type[%s] fail", type.c_str()); for (const auto &it : attrs) { GE_CHK_BOOL_EXEC(model.SetAttr("ATTR_MODEL_" + it.first, it.second) == GRAPH_SUCCESS, return FAILED, - "Set OpDesc attribute[%s] fail", it.first.c_str()); + "[Set][Attr] OpDesc attribute[%s] fail", it.first.c_str()); } GE_CHK_BOOL_EXEC(ge::AttrUtils::SetListTensor(&model, "ATTR_MODEL_TENSOR_INPUTS", inputs), return FAILED, - "Set Inputs tensor list fail"); + "[Set][InputsTensor] list fail"); GE_CHK_BOOL_EXEC(ge::AttrUtils::SetListTensor(&model, "ATTR_MODEL_TENSOR_OUTPUTS", outputs), return FAILED, - "Set Outputs tensor list fail"); + "[Set][OutputsTensor] list fail"); return SUCCESS; } @@ -1645,9 +1643,8 @@ Status GraphManager::RemoveGraph(const GraphId &graph_id) { GraphNodePtr graph_node = nullptr; Status ret = GetGraphNode(graph_id, graph_node); if (ret != SUCCESS || graph_node == nullptr) { - REPORT_INNER_ERROR("E19999", "Graph:%u not exist in graph_map, check invalid when GraphManager %s", - graph_id, __FUNCTION__); - GELOGE(GE_GRAPH_GRAPH_NOT_EXIST, "[GraphManager] Id %u does not exists.", graph_id); + REPORT_INNER_ERROR("E19999", "Graph:%u not exist in graph_map, check invalid", graph_id); + GELOGE(GE_GRAPH_GRAPH_NOT_EXIST, "[Get][GraphNode] Id %u does not exists.", graph_id); return GE_GRAPH_GRAPH_NOT_EXIST; } if (graph_node->GetRunFlag()) { @@ -1672,7 +1669,7 @@ Status GraphManager::RemoveGraph(const GraphId &graph_id) { if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtSetDevice failed, device_id:%u, graph_id:%u", GetContext().DeviceId(), graph_id); - GELOGE(RT_FAILED, "[GraphManager:] rtSetDevice failed, modelId=%u, graphId=%u.", ge_root_model->GetModelId(), + GELOGE(RT_FAILED, "[Call][RtSetDevice] failed, modelId=%u, graphId=%u.", ge_root_model->GetModelId(), graph_id); return FAILED; } @@ -1680,16 +1677,15 @@ Status GraphManager::RemoveGraph(const GraphId &graph_id) { // unload them respectively. middle_ret = UnloadModel(ge_root_model, graph_id); if (middle_ret != SUCCESS) { - REPORT_INNER_ERROR("E19999", "UnloadModel for graph:%u failed, check unload detail in GraphLoader %s", - graph_id, __FUNCTION__); - GELOGE(middle_ret, "[GraphManager:] unload model failed, graph_id=%u.", graph_id); + REPORT_INNER_ERROR("E19999", "UnloadModel for graph:%u failed, check invalid", graph_id); + GELOGE(middle_ret, "[Unload][Model] model failed, graph_id=%u.", graph_id); ret = middle_ret; } rt_ret = rtDeviceReset(GetContext().DeviceId()); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtDeviceReset failed, device_id:%u, graph_id:%u, when GraphManager %s", - GetContext().DeviceId(), graph_id, __FUNCTION__); - GELOGE(RT_FAILED, "[GraphManager:] rtDeviceReset failed, graphId=%u.", graph_id); + REPORT_CALL_ERROR("E19999", "Call rtDeviceReset failed, device_id:%u, graph_id:%u", + GetContext().DeviceId(), graph_id); + GELOGE(RT_FAILED, "[Call][RtDeviceReset] failed, device_id:%u, graph_id:%u", GetContext().DeviceId(), graph_id); ret = FAILED; } } @@ -1698,7 +1694,7 @@ Status GraphManager::RemoveGraph(const GraphId &graph_id) { RemoveGraphCount(graph_id); RemoveAddGraphCondition(graph_id); - GE_CHK_STATUS_RET(ret, "[GraphManager:] Remove graph failed, graph_id=%u.", graph_id); + GE_CHK_STATUS_RET(ret, "[Remove][Graph] failed, graph_id=%u.", graph_id); GELOGI("[GraphManager] remove graph success, graph_id=%u.", graph_id); return SUCCESS; } @@ -1711,8 +1707,7 @@ Status GraphManager::ParseOptions(const std::map &opti ret = ParseOption(options, STREAM_MAX_PARALLEL_NUM, options_.stream_max_parallel_num); if (ret != SUCCESS) { GELOGE(GE_GRAPH_OPTIONS_INVALID, - "parse Key:%s value failed, it must be same format as " - "DNN_V100:2,DNN_HCCL:3", + "[Parse][Option] %s value failed, it must be same format as DNN_V100:2,DNN_HCCL:3", STREAM_MAX_PARALLEL_NUM.c_str()); return GE_GRAPH_OPTIONS_INVALID; } @@ -1720,23 +1715,23 @@ Status GraphManager::ParseOptions(const std::map &opti // get stream num ret = ParseOption(options, STREAM_NUM, options_.stream_num); if ((ret != SUCCESS) || (options_.stream_num == 0)) { - GELOGE(GE_GRAPH_OPTIONS_INVALID, "Key:ge.stream_num, its value %d is invalid, must be not equal zero.", - options_.stream_num); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Parse][Option] Key:ge.stream_num, its value %d is invalid, " + "must be not equal zero.", options_.stream_num); return GE_GRAPH_OPTIONS_INVALID; } // get perf level, its value please see enum PerfLevel ret = ParseOption(options, PERF_LEVEL, options_.perf_level); if ((ret != SUCCESS) || IsPerfLevelInvalid(options_.perf_level)) { - GELOGE(GE_GRAPH_OPTIONS_INVALID, "Key:ge.perfLevel, its value %d is invalid, must be enum PerfLevel type.", - options_.perf_level); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Parse][Option] Key:ge.perfLevel, its value %d is invalid, " + "must be enum PerfLevel type.", options_.perf_level); return GE_GRAPH_OPTIONS_INVALID; } // get encrypt mode ret = ParseOption(options, ENCRYPT_MODE, options_.encrypt_mode); GE_IF_BOOL_EXEC(ret != SUCCESS, - GELOGE(GE_GRAPH_OPTIONS_INVALID, "Key:ge.encryptMode value invalid."); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Parse][Option] Key:ge.encryptMode value invalid."); return GE_GRAPH_OPTIONS_INVALID); // get ek file @@ -1776,7 +1771,8 @@ Status GraphManager::ParseOptions(const std::map &opti // get weight compress flag ret = ParseOption(options, COMPRESS_FLAG, options_.compress_flag); GE_IF_BOOL_EXEC(ret != SUCCESS, - GELOGE(GE_GRAPH_OPTIONS_INVALID, "Key:ge.compressFlag value is invalid, must be 0 or 1."); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Parse][Option] Key:ge.compressFlag value is invalid, " + "must be 0 or 1."); return GE_GRAPH_OPTIONS_INVALID); // Set Build model and step ParseOption(options, BUILD_MODE, options_.build_mode); @@ -1787,21 +1783,22 @@ Status GraphManager::ParseOptions(const std::map &opti options_.run_graph_flag = true; ret = ParseOption(options, RUN_FLAG, options_.run_graph_flag); GE_IF_BOOL_EXEC(ret != SUCCESS, - GELOGE(GE_GRAPH_OPTIONS_INVALID, "Key:ge.runFlag value is invalid, must be 0 or 1."); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Parse][Option] Key:ge.runFlag value is invalid, must be 0 or 1."); return GE_GRAPH_OPTIONS_INVALID); // ge.graphType ret = ParseTrainGraphFlag(options_.run_graph_flag, options_.train_graph_flag); GE_IF_BOOL_EXEC(ret != SUCCESS, - GELOGE(GE_GRAPH_OPTIONS_INVALID, "Key:ge.runFlag value is invalid"); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Parse][TrainGraphFlag] Key:ge.runFlag value is invalid"); return GE_GRAPH_OPTIONS_INVALID); // parse FmkOp options_.local_fmk_op_flag = false; ret = ParseOption(options, LOCAL_FMKOP_FLAG, options_.local_fmk_op_flag); GE_IF_BOOL_EXEC(ret != SUCCESS, - GELOGE(GE_GRAPH_OPTIONS_INVALID, "Key:ge.localFmkopFlag value is invalid, must be 0 or 1."); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Parse][Option] Key:ge.localFmkopFlag value is invalid, " + "must be 0 or 1."); return GE_GRAPH_OPTIONS_INVALID); options_.enable_print_op_pass = true; ret = ParseOption(options, ENABLE_PRINT_OP_PASS, options_.enable_print_op_pass); @@ -1809,13 +1806,15 @@ Status GraphManager::ParseOptions(const std::map &opti options_.is_single_op = false; ret = ParseOption(options, SINGLE_OP_FLAG, options_.is_single_op); GE_IF_BOOL_EXEC(ret != SUCCESS, - GELOGE(GE_GRAPH_OPTIONS_INVALID, "Key:ge.enablePrintOpPass value is invalid, must be 0 or 1."); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Parse][Option] Key:ge.enablePrintOpPass value is invalid, " + "must be 0 or 1."); return GE_GRAPH_OPTIONS_INVALID); // parse hcom parallel options_.hcom_parallel = false; ret = ParseOption(options, HCOM_PARALLEL, options_.hcom_parallel); GE_IF_BOOL_EXEC(ret != SUCCESS, - GELOGE(GE_GRAPH_OPTIONS_INVALID, "Key:ge.hcomParallel value is invalid, must be 0 or 1."); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Parse][Option] Key:ge.hcomParallel value is invalid, " + "must be 0 or 1."); return GE_GRAPH_OPTIONS_INVALID); // net output node dataType ParseOption(options, OUTPUT_DATATYPE, options_.output_datatype); @@ -1875,10 +1874,9 @@ Status GraphManager::ParseOption(const std::map &optio } else if (flag == "1") { option = true; } else { - REPORT_INNER_ERROR("E19999", "Option:%s value:%s must be 0 or 1, check invalid", - key.c_str(), flag.c_str()); - GELOGE(GE_GRAPH_OPTIONS_INVALID, "Key:%s, its value %s is invalid, it must be 0 or 1.", key.c_str(), - flag.c_str()); + REPORT_INNER_ERROR("E19999", "Option:%s value:%s must be 0 or 1, check invalid", key.c_str(), flag.c_str()); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Check][Param] Key:%s, its value %s is invalid, it must be 0 or 1.", + key.c_str(), flag.c_str()); return GE_GRAPH_OPTIONS_INVALID; } } @@ -1895,8 +1893,8 @@ Status GraphManager::ParseOption(const std::map &optio if (ptr != nullptr && *ptr != '\0') { REPORT_INNER_ERROR("E19999", "Option:%s value:%s must be int32_t type, check invalid", key.c_str(), iter->second.c_str()); - GELOGE(GE_GRAPH_OPTIONS_INVALID, "Key:%s, its value %s is invalid, must be int32_t type.", key.c_str(), - iter->second.c_str()); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Check][Param] Key:%s, its value %s is invalid, must be int32_t type.", + key.c_str(), iter->second.c_str()); return GE_GRAPH_OPTIONS_INVALID; } } @@ -1940,10 +1938,8 @@ Status GraphManager::ParseOption(const std::map &optio if (pos == string::npos) { REPORT_INNER_ERROR("E19999", "Option:%s, value:%s, engine and num must be connected by :, check invalid", key.c_str(), engine_parallel.c_str()); - GELOGE(GE_GRAPH_OPTIONS_INVALID, - "engine and num must be connected by :, " - "while your input is %s", - engine_parallel.c_str()); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Check][Param] engine and num must be connected by :, " + "while your input is %s", engine_parallel.c_str()); return GE_GRAPH_OPTIONS_INVALID; } std::string engine_name = engine_parallel.substr(0, pos); @@ -1953,14 +1949,14 @@ Status GraphManager::ParseOption(const std::map &optio Status ret = CheckEngineName(engine_name, key, option); if (ret != SUCCESS) { - GELOGE(GE_GRAPH_OPTIONS_INVALID, "check engine name : %s failed, ", engine_name.c_str()); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Check][EngineName] %s failed", engine_name.c_str()); return GE_GRAPH_OPTIONS_INVALID; } int num = 0; ret = ParseParallelNum(parallel_num, key, num); if (ret != SUCCESS) { - GELOGE(GE_GRAPH_OPTIONS_INVALID, "parse parallel num failed"); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Parse][ParallelNum] %s failed", parallel_num.c_str()); return GE_GRAPH_OPTIONS_INVALID; } @@ -1975,7 +1971,7 @@ Status GraphManager::CheckEngineName(const std::string &engine_name, const std:: if (engine_name.empty()) { REPORT_INNER_ERROR("E19999", "Option:%s, param engine_name:%s is empty, check invalid", key.c_str(), engine_name.c_str()); - GELOGE(GE_GRAPH_OPTIONS_INVALID, "engine name of %s is empty", key.c_str()); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Check][Param] engine name of %s is empty", key.c_str()); return GE_GRAPH_OPTIONS_INVALID; } // judge whether exist in engine list @@ -1987,7 +1983,7 @@ Status GraphManager::CheckEngineName(const std::string &engine_name, const std:: if (it_stream_repeat != option.end()) { REPORT_INNER_ERROR("E19999", "Option:%s, param engine_name:%s is repeated, check invalid", key.c_str(), engine_name.c_str()); - GELOGE(GE_GRAPH_OPTIONS_INVALID, "engine : %s of %s is repeated", engine_name.c_str(), key.c_str()); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Check][Param] engine:%s of %s is repeated", engine_name.c_str(), key.c_str()); return GE_GRAPH_OPTIONS_INVALID; } return SUCCESS; @@ -1997,14 +1993,15 @@ Status GraphManager::ParseParallelNum(const std::string ¶llel_num, const std if (parallel_num.empty()) { REPORT_INNER_ERROR("E19999", "Option:%s, param parallel num:%s is empty, check invalid", key.c_str(), parallel_num.c_str()); - GELOGE(GE_GRAPH_OPTIONS_INVALID, "parallel num of %s is empty", key.c_str()); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Check][Param] parallel num of %s is empty", key.c_str()); return GE_GRAPH_OPTIONS_INVALID; } for (char c : parallel_num) { if (!isdigit(c)) { REPORT_INNER_ERROR("E19999", "Option:%s, param parallel num:%s is not digit, check invalid", key.c_str(), parallel_num.c_str()); - GELOGE(GE_GRAPH_OPTIONS_INVALID, "%s input is invalid ", key.c_str()); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Check][Param] Option:%s, param parallel num:%s is not digit, check invalid", + key.c_str(), parallel_num.c_str()); return GE_GRAPH_OPTIONS_INVALID; } } @@ -2014,24 +2011,28 @@ Status GraphManager::ParseParallelNum(const std::string ¶llel_num, const std } catch (std::invalid_argument &) { REPORT_INNER_ERROR("E19999", "Option:%s, param parallel num:%s is invalid argument, check", key.c_str(), parallel_num.c_str()); - GELOGE(GE_GRAPH_OPTIONS_INVALID, "parallel num : %s of %s is invalid argument", parallel_num.c_str(), key.c_str()); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Check][Param] parallel num:%s of %s is invalid argument", + parallel_num.c_str(), key.c_str()); return GE_GRAPH_OPTIONS_INVALID; } catch (std::out_of_range &) { REPORT_INNER_ERROR("E19999", "Option:%s, param parallel num:%s is out of range, check", key.c_str(), parallel_num.c_str()); - GELOGE(GE_GRAPH_OPTIONS_INVALID, "parallel num : %s of %s is out of range", parallel_num.c_str(), key.c_str()); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Check][Param] parallel num:%s of %s is out of range", + parallel_num.c_str(), key.c_str()); return GE_GRAPH_OPTIONS_INVALID; } catch (...) { REPORT_INNER_ERROR("E19999", "Option:%s, param parallel num:%s is invalid argument, check", key.c_str(), parallel_num.c_str()); - GELOGE(GE_GRAPH_OPTIONS_INVALID, "parallel num : %s of %s is invalid argument", parallel_num.c_str(), key.c_str()); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Check][Param] parallel num:%s of %s is invalid argument", + parallel_num.c_str(), key.c_str()); return GE_GRAPH_OPTIONS_INVALID; } if (num < 1) { REPORT_INNER_ERROR("E19999", "Option:%s, param parallel num:%s < 1, check invalid", key.c_str(), parallel_num.c_str()); - GELOGE(GE_GRAPH_OPTIONS_INVALID, "parallel num : %s of %s must bigger than 0", parallel_num.c_str(), key.c_str()); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Check][Param] parallel num:%s of %s must bigger than 0", + parallel_num.c_str(), key.c_str()); return GE_GRAPH_OPTIONS_INVALID; } return SUCCESS; @@ -2058,9 +2059,8 @@ Status GraphManager::GetGraphNode(const GraphId &graph_id, GraphNodePtr &out) { auto iter = graph_map_.find(graph_id); if (iter == graph_map_.end()) { out = nullptr; - REPORT_INNER_ERROR("E19999", "Graph:%u not exist in graph_map, check invalid", - graph_id); - GELOGE(GE_GRAPH_GRAPH_NOT_EXIST, "[GraphManager] graph not exist, graph_id= %u.", graph_id); + REPORT_INNER_ERROR("E19999", "Graph:%u not exist in graph_map, check invalid", graph_id); + GELOGE(GE_GRAPH_GRAPH_NOT_EXIST, "[Check][Param] graph not exist, graph_id= %u.", graph_id); return GE_GRAPH_GRAPH_NOT_EXIST; } out = iter->second; @@ -2081,7 +2081,7 @@ Status GraphManager::SummaryHandle(const GraphId &graph_id, std::vector &summary_output_indexes = whole_summary_output_indexes.at(graph_id); @@ -2126,9 +2126,8 @@ Status GraphManager::CheckpointHandle(const GraphId &graph_id, const ComputeGrap } } if (netoutput == nullptr) { - REPORT_INNER_ERROR("E19999", "No netoutput node in graph:%u, check invalid", - graph_id); - GELOGE(FAILED, "Netoutput is null."); + REPORT_INNER_ERROR("E19999", "No netoutput node in graph:%u, check invalid", graph_id); + GELOGE(FAILED, "[Check][Param] No netoutput node in graph:%u", graph_id); return FAILED; } for (const auto &in : netoutput->GetAllInDataAnchors()) { @@ -2136,9 +2135,9 @@ Status GraphManager::CheckpointHandle(const GraphId &graph_id, const ComputeGrap auto out_anchor = in->GetPeerOutAnchor(); if (out_anchor == nullptr) { REPORT_INNER_ERROR("E19999", "Peer anchor of op:%s(%s), in_index:%u is nullptr, graph_id:%u, check invalid", - netoutput->GetName().c_str(), netoutput->GetType().c_str(), - in->GetIdx(), graph_id); - GELOGE(FAILED, "out_anchor is null."); + netoutput->GetName().c_str(), netoutput->GetType().c_str(), in->GetIdx(), graph_id); + GELOGE(FAILED, "[Get][PeerOutAnchor] Peer anchor of op:%s(%s), in_index:%u is nullptr, graph_id:%u", + netoutput->GetName().c_str(), netoutput->GetType().c_str(), in->GetIdx(), graph_id); return FAILED; } ge::NodePtr peer_node = out_anchor->GetOwnerNode(); @@ -2147,7 +2146,8 @@ Status GraphManager::CheckpointHandle(const GraphId &graph_id, const ComputeGrap if (peer_node->GetAllInDataAnchors().size() != 1) { REPORT_INNER_ERROR("E19999", "More than one prior nodes of peer_node:%s(%s) in checkpoint Graph:%u, " "check invalid", peer_node->GetName().c_str(), peer_node->GetType().c_str(), graph_id); - GELOGE(FAILED, "More than one prior nodes of peer_node %s in checkpoint Graph.", peer_node->GetName().c_str()); + GELOGE(FAILED, "[Check][Param] More than one prior nodes of peer_node:%s(%s) in checkpoint Graph:%u.", + peer_node->GetName().c_str(), peer_node->GetType().c_str(), graph_id); return FAILED; } auto peer_node_in = peer_node->GetAllInDataAnchors().at(0); @@ -2161,9 +2161,9 @@ Status GraphManager::CheckpointHandle(const GraphId &graph_id, const ComputeGrap } if (peer_node == nullptr) { REPORT_INNER_ERROR("E19999", "Peer anchor node of op:%s(%s), in_index:%u is nullptr, graph_id:%u, check invalid", - netoutput->GetName().c_str(), netoutput->GetType().c_str(), - in->GetIdx(), graph_id); - GELOGE(FAILED, "No variable op found in one branch, checkpoint graph illegal."); + netoutput->GetName().c_str(), netoutput->GetType().c_str(), in->GetIdx(), graph_id); + GELOGE(FAILED, "[Check][Param] Peer anchor node of op:%s(%s), in_index:%u is nullptr, graph_id:%u", + netoutput->GetName().c_str(), netoutput->GetType().c_str(), in->GetIdx(), graph_id); return FAILED; } desc_name = peer_node->GetName(); @@ -2172,7 +2172,8 @@ Status GraphManager::CheckpointHandle(const GraphId &graph_id, const ComputeGrap REPORT_INNER_ERROR("E19999", "in index:%u of op:%s(%s) is out of outputs.size:%zu range, graph_id:%u, " "check invalid", in->GetIdx(), netoutput->GetName().c_str(), netoutput->GetType().c_str(), outputs.size(), graph_id); - GELOGE(FAILED, "variable index out of range."); + GELOGE(FAILED, "[Check][Param] in index:%u of op:%s(%s) is out of outputs.size:%zu range, graph_id:%u", + in->GetIdx(), netoutput->GetName().c_str(), netoutput->GetType().c_str(), outputs.size(), graph_id); return FAILED; } save_results.emplace(desc_name, TensorAdapter::AsTensor(outputs.at(in->GetIdx()))); @@ -2218,9 +2219,8 @@ Status GraphManager::PushSummaryData2ME(const GraphId &graph_id, } return iter->second(graph_id, tmp_summary_data); } - REPORT_INNER_ERROR("E19999", "No summary callback found, graph_id:%u, check invalid", - graph_id); - GELOGE(FAILED, "[GraphManager] PushSummaryData2ME failed, not found summary callback."); + REPORT_INNER_ERROR("E19999", "No summary callback found, graph_id:%u, check invalid", graph_id); + GELOGE(FAILED, "[Check][Param] No summary callback found, graph_id:%u", graph_id); return FAILED; } return itr->second(graph_id, summary_data); @@ -2240,9 +2240,8 @@ Status GraphManager::PushSaveData2ME(const GraphId &graph_id, const std::mapsecond(graph_id, tmp_save_data); } - REPORT_INNER_ERROR("E19999", "No checkpoint callback found, graph_id:%u, check invalid", - graph_id); - GELOGE(FAILED, "[GraphManager] PushSaveData2ME failed, not found checkpoint callback."); + REPORT_INNER_ERROR("E19999", "No checkpoint callback found, graph_id:%u, check invalid", graph_id); + GELOGE(FAILED, "[Check][Param] No checkpoint callback found, graph_id:%u", graph_id); return FAILED; } return itr->second(graph_id, save_data); @@ -2272,7 +2271,8 @@ bool GraphManager::CheckVariableForCheckpointGraph(NodePtr &node) { if (out == nullptr) { REPORT_INNER_ERROR("E19999", "anchor index:0 of op:%s(%s) is nullptr, check invalid", node->GetName().c_str(), node->GetType().c_str()); - GELOGE(GE_GRAPH_PARAM_NULLPTR, "out is nullptr."); + GELOGE(GE_GRAPH_PARAM_NULLPTR, "[Get][OutDataAnchor] anchor index:0 of op:%s(%s) is nullptr", + node->GetName().c_str(), node->GetType().c_str()); return false; } auto peer_out = out->GetPeerInDataAnchors(); @@ -2305,7 +2305,7 @@ static inline bool CheckConstanOpForCheckpointGraph(NodePtr &node) { return node bool GraphManager::IsCheckpointGraph(ComputeGraphPtr &compute_graph) { if (compute_graph == nullptr) { REPORT_INNER_ERROR("E19999", "Param compute_graph is nullptr, check invalid"); - GELOGE(GE_GRAPH_PARAM_NULLPTR, "[IsCheckpointGraph] computeGraph is nullptr."); + GELOGE(GE_GRAPH_PARAM_NULLPTR, "[Check][Param] computeGraph is nullptr."); return false; } for (auto &node : compute_graph->GetAllNodes()) { @@ -2439,9 +2439,8 @@ Status GraphManager::RemoveIsolatedConstInThisGraph(ge::ComputeGraphPtr &compute if (n->GetOutAllNodes().empty() && n->GetInAllNodes().empty()) { // it is an isolated constant, just remove it if (GraphUtils::RemoveJustNode(compute_graph, n) != GRAPH_SUCCESS) { - REPORT_CALL_ERROR("E19999", "Remove constant op:%s(%s) failed", - n->GetName().c_str(), n->GetType().c_str()); - GELOGE(FAILED, "remove constant %s failed.", n->GetName().c_str()); + REPORT_CALL_ERROR("E19999", "Remove constant op:%s(%s) failed", n->GetName().c_str(), n->GetType().c_str()); + GELOGE(FAILED, "[Call][RemoveJustNode] remove constant %s failed.", n->GetName().c_str()); return FAILED; } } @@ -2520,7 +2519,7 @@ Status GraphManager::OptimizeStage1(ge::ComputeGraphPtr &compute_graph) { auto ret = after_merge_passes.Run(compute_graph); GE_TIMESTAMP_END(after_merge_passes, "GraphManager::OptimizeStage1_1"); if (ret != SUCCESS && ret != NOT_CHANGED) { - GELOGE(ret, "Run passes when OptimizeStage1_1 failed, ret:%u.", ret); + GELOGE(ret, "[Run][Passes] when OptimizeStage1_1 failed, ret:%u.", ret); return ret; } @@ -2559,7 +2558,7 @@ Status GraphManager::OptimizeStage1(ge::ComputeGraphPtr &compute_graph) { ret = GEPass(compute_graph).Run(names_to_passes); GE_TIMESTAMP_END(names_to_passes, "GraphManager::OptimizeStage1_2"); if (ret != SUCCESS) { - GELOGE(ret, "Run passes when OptimizeStage1_2 failed, ret:%u.", ret); + GELOGE(ret, "[Run][Passes] when OptimizeStage1_2 failed, ret:%u.", ret); return ret; } // Calculate Op/Fe constantfolding cost @@ -2582,8 +2581,6 @@ Status GraphManager::OptimizeStage1(ge::ComputeGraphPtr &compute_graph) { GE_CHK_STATUS_RET(graph_pass.AddPass("OptimizeStage1_3::Migration", new (std::nothrow) SubgraphConstMigrationPass)); GE_CHK_STATUS_RET(graph_pass.AddPass("OptimizeStage1_3::ArgsClean", new (std::nothrow) UnusedArgsCleanPass)); GE_CHK_STATUS_RET(graph_pass.AddPass("OptimizeStage1_3::PrunePass", new (std::nothrow) PrunePass)); - auto mark_force_unknown_pass = new (std::nothrow) MarkForceUnknownForCondPass; - GE_CHK_STATUS_RET(graph_pass.AddPass("OptimizeStage1_3::MarkForceUnknownForCondPass", mark_force_unknown_pass)); GE_CHK_STATUS_RET(graph_pass.AddPass("OptimizeStage1_3::NextIterationPass", new (std::nothrow) NextIterationPass)) GE_CHK_STATUS_RET(graph_pass.AddPass("OptimizeStage1_3::ControlTriggerPass", new (std::nothrow) ControlTriggerPass)) GE_CHK_STATUS_RET( @@ -2617,7 +2614,7 @@ Status GraphManager::OptimizeStage1(ge::ComputeGraphPtr &compute_graph) { ret = graph_pass.Run(compute_graph); GE_TIMESTAMP_END(graph_pass, "GraphManager::OptimizeStage1_3"); if (ret != SUCCESS && ret != NOT_CHANGED) { - GELOGE(ret, "Run passes when OptimizeStage1_3 failed, ret:%u.", ret); + GELOGE(ret, "[Run][Passes] when OptimizeStage1_3 failed, ret:%u.", ret); return ret; } NamesToPass node_pass; @@ -2627,7 +2624,7 @@ Status GraphManager::OptimizeStage1(ge::ComputeGraphPtr &compute_graph) { ret = GEPass(compute_graph).Run(node_pass); GE_TIMESTAMP_END(node_pass, "GraphPrepare::node_pass"); if (ret != SUCCESS) { - GELOGE(ret, "Run identity remove pass for preprocess failed, ret:%u.", ret); + GELOGE(ret, "[Run][Identity] remove pass for preprocess failed, ret:%u.", ret); return ret; } return SUCCESS; @@ -2647,7 +2644,7 @@ Status GraphManager::OptimizeStage2(ge::ComputeGraphPtr &compute_graph) { auto ret = after_merge_passes.Run(compute_graph); GE_TIMESTAMP_END(after_merge_passes, "OptimizeStage2::AfterMergePasses"); if (ret != SUCCESS && ret != NOT_CHANGED) { - GELOGE(ret, "Run passes after merge sub graph failed, ret:%d.", ret); + GELOGE(ret, "[Run][Passes] after merge sub graph failed, ret:%d.", ret); return ret; } SetAttrForHcomBroadCastOp(compute_graph); @@ -2671,13 +2668,13 @@ Status GraphManager::OptimizeStage2(ge::ComputeGraphPtr &compute_graph) { ret = GEPass(compute_graph).Run(names_to_passes); GE_TIMESTAMP_END(names_to_passes, "OptimizeStage2::MergedGraphNameToPasses"); if (ret != SUCCESS) { - GELOGE(ret, "Run ge_passes optimize for OptimizeAfterMergeSubGraph failed, ret:%d.", ret); + GELOGE(ret, "[Run][GEPasses] optimize for OptimizeAfterMergeSubGraph failed, ret:%d.", ret); return ret; } ret = RemoveIsolatedConst(compute_graph); if (ret != SUCCESS) { - GELOGE(ret, "Remove isolated Constant failed, ret:%d.", ret); + GELOGE(ret, "[Remove][IsolatedConst] failed, ret:%d.", ret); return ret; } @@ -2728,32 +2725,33 @@ Status GraphManager::OptimizeStage2(ge::ComputeGraphPtr &compute_graph) { ret = pass_for_control_attr_optimize.Run(compute_graph); GE_TIMESTAMP_END(pass_for_control_attr_optimize, "OptimizeStage2::ControlAttrOptimize"); if (ret != SUCCESS && ret != NOT_CHANGED) { - GELOGE(ret, "Run passes when optimize stage 2 failed"); + GELOGE(ret, "[Run][Passes] when optimize stage 2 failed"); return ret; } // Assign functional op labels. GE_TIMESTAMP_START(AssignFunctionalLabels); LabelAllocator label_allocator(compute_graph); - GE_CHK_STATUS_RET(label_allocator.AssignFunctionalLabels(), "Assign label failed."); + GE_CHK_STATUS_RET(label_allocator.AssignFunctionalLabels(), "[Assign][Label] failed."); GE_TIMESTAMP_END(AssignFunctionalLabels, "ModelBuilder::AssignFunctionalLabels"); // Add memcpy addr asynchronous node. GE_TIMESTAMP_START(AddMemcpyAddrAsyncNode); MemcpyAddrAsyncPass memcpy_addr; - GE_CHK_STATUS_RET(memcpy_addr.Run(compute_graph), "Add memcpy_addr_async node failed."); + GE_CHK_STATUS_RET(memcpy_addr.Run(compute_graph), "[Call][Run] Add memcpy_addr_async node failed."); GE_TIMESTAMP_END(AddMemcpyAddrAsyncNode, "MemcpyAddrAsyncPass::Run."); // Process offset and dependency for buffer pool memory assigner. GE_TIMESTAMP_START(BufferPoolMemoryPass); BufferPoolMemoryPass buffer_pool_mem_pass; - GE_CHK_STATUS_RET(buffer_pool_mem_pass.Run(compute_graph), "Failed to process for buffer pool allocator."); + GE_CHK_STATUS_RET(buffer_pool_mem_pass.Run(compute_graph), + "[Call][Run] Failed to process for buffer pool allocator."); GE_TIMESTAMP_END(BufferPoolMemoryPass, "BufferPoolMemoryPass::Run."); // Handle parallel group . GE_TIMESTAMP_START(ParallelGroup); ParallelGroupPass parallel_group_pass; - GE_CHK_STATUS_RET(parallel_group_pass.Run(compute_graph), "Handle parallel group failed."); + GE_CHK_STATUS_RET(parallel_group_pass.Run(compute_graph), "[Handle][ParallelGroup] failed."); GE_TIMESTAMP_END(ParallelGroup, "ParallelGroupPass::Run."); // After while sub graph handle, mark all node rw type @@ -2806,7 +2804,7 @@ Status GraphManager::LoadGraphAsync(const GeRootModelPtr &ge_root_model, const G Status ret = GraphLoader::LoadModelOnline(model_id_info.model_id, ge_root_model, listener); GE_TIMESTAMP_EVENT_END(LoadGraph, "GraphManager::LoadGraphAsync"); if (ret != SUCCESS) { - GELOGE(ret, "[LoadGraphAsync] LoadGraphAsync Failed"); + GELOGE(ret, "[Load][ModelOnline] Failed, model_id:%u", model_id_info.model_id); graph_node->SetRunFlag(false); return ret; } @@ -2821,9 +2819,8 @@ void GraphManager::ReleaseMemory(const GeModelPtr &ge_model, GraphNodePtr &graph const std::vector &model_ids, uint32_t graph_id, uint64_t session_id) { rtError_t rt_ret = rtSetDevice(GetContext().DeviceId()); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtSetDevice failed, device_id:%u, when GraphManager %s", - GetContext().DeviceId(), __FUNCTION__); - GELOGE(RT_FAILED, "[GraphManager:] rtSetDevice failed, graphId=%u.", graph_id); + REPORT_CALL_ERROR("E19999", "Call rtSetDevice failed, device_id:%u", GetContext().DeviceId()); + GELOGE(RT_FAILED, "[Call][RtSetDevice] failed, device_id=%u.", GetContext().DeviceId()); return; } for (auto model_id : model_ids) { @@ -2865,9 +2862,8 @@ void GraphManager::ReleaseMemory(const GeModelPtr &ge_model, GraphNodePtr &graph ge_root_model->ClearAllModelId(); rt_ret = rtDeviceReset(GetContext().DeviceId()); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtDeviceReset failed, device_id:%u, when GraphManager %s", - GetContext().DeviceId(), __FUNCTION__); - GELOGE(RT_FAILED, "[GraphManager:] rtDeviceReset failed, graphId=%u.", graph_id); + REPORT_CALL_ERROR("E19999", "Call rtDeviceReset failed, device_id:%u", GetContext().DeviceId()); + GELOGE(RT_FAILED, "[Call][RtDeviceReset] failed, device_id:%u.", GetContext().DeviceId()); return; } } @@ -2895,7 +2891,8 @@ Status GraphManager::CheckAndReleaseMemory(const GeModelPtr &ge_model, const Gra if (ge::CheckInt64AddOverflow(memory_size, weight_size) != SUCCESS) { REPORT_INNER_ERROR("E19999", "memory_size:%ld and weight_size:%ld will overflow after add, check invalid", memory_size, weight_size); - GELOGE(INTERNAL_ERROR, "The sum of Memory size and weight size exceeds INT64_MAX"); + GELOGE(INTERNAL_ERROR, "[Check][Param] memory_size:%ld and weight_size:%ld will overflow after add", + memory_size, weight_size); return INTERNAL_ERROR; } if (free_memory >= (memory_size + weight_size)) { @@ -2955,23 +2952,25 @@ Status GraphManager::ProcessSubGraphWithMultiThreads(GraphManager *graph_manager GE_DUMP(compute_graph_tmp, "OptimizeSubGraphBefore"); GE_CHECK_NOTNULL(compute_graph_tmp); if (!AttrUtils::SetInt(*compute_graph_tmp, ATTR_NAME_ROOT_GRAPH_ID, root_graph_id)) { - REPORT_CALL_ERROR("E19999", "Set Attr:%s to graph:%u", ATTR_NAME_ROOT_GRAPH_ID.c_str(), + REPORT_CALL_ERROR("E19999", "Set Attr:%s to graph:%u failed", ATTR_NAME_ROOT_GRAPH_ID.c_str(), compute_graph_tmp->GetGraphID()); - GELOGE(FAILED, "Failed to set attr ATTR_NAME_ROOT_GRAPH_ID for subgraph, graph_id: %u.", root_graph_id); + GELOGE(FAILED, "[Set][Attr] %s to graph:%u failed", ATTR_NAME_ROOT_GRAPH_ID.c_str(), + compute_graph_tmp->GetGraphID()); return FAILED; } if (!AttrUtils::SetStr(*compute_graph_tmp, ATTR_NAME_ROOT_GRAPH_NAME, root_graph_name)) { - REPORT_CALL_ERROR("E19999", "Set Attr:%s to graph:%u", ATTR_NAME_ROOT_GRAPH_NAME.c_str(), + REPORT_CALL_ERROR("E19999", "Set Attr:%s to graph:%u failed", ATTR_NAME_ROOT_GRAPH_NAME.c_str(), compute_graph_tmp->GetGraphID()); - GELOGE(FAILED, "Failed to set attr ATTR_NAME_ROOT_GRAPH_NAME for subgraph, \ - root_graph_name: %s.", root_graph_name.c_str()); + GELOGE(FAILED, "[Set][Attr] %s to graph:%u failed", ATTR_NAME_ROOT_GRAPH_NAME.c_str(), + compute_graph_tmp->GetGraphID()); return FAILED; } compute_graph_tmp->SetSessionID(session_id); Status ret = graph_manager->GetCompilerStages(root_graph_id).optimizer.OptimizeSubGraph(compute_graph_tmp, engine_name); if (ret != SUCCESS) { - GELOGE(ret, "SubGraph optimize Failed %s", engine_name.c_str()); + GELOGE(ret, "[Optimize][SubGraph] Failed, engine:%s, graph:%s", + engine_name.c_str(), compute_graph_tmp->GetName().c_str()); return ret; } else { GELOGD("SubGraph optimize success %s", engine_name.c_str()); @@ -2983,7 +2982,7 @@ Status GraphManager::ProcessSubGraphWithMultiThreads(GraphManager *graph_manager pthread_self()); } else { REPORT_INNER_ERROR("E19999", "Param sub_graph_info_ptr or graph_manager is nullptr"); - GELOGE(FAILED, "graph_manager or sub_graph_info_ptr is nullptr"); + GELOGE(FAILED, "[Check][Param] graph_manager or sub_graph_info_ptr is nullptr"); return FAILED; } @@ -3000,7 +2999,7 @@ Status GraphManager::RunGraphAsync(const GraphId &graph_id, const std::vectorGetGraphId()) + " need to re-build, you should remove it" " from GE first, then AddGraph again and rebuild it."); - graph_node->Unlock(); return PARAM_INVALID; } // check need incre build. @@ -3121,14 +3119,14 @@ void GraphManager::PreRunThread(GraphManager *graph_manager) { GraphNodePtr graph_node = nullptr; Status ret = graph_manager->GetGraphNode(args.graph_id, graph_node); if (ret != SUCCESS) { - ReturnError(graph_manager, args.callback, GE_GRAPH_ALREADY_RUNNING, + ReturnError(graph_manager, args.callback, GE_GRAPH_GRAPH_NODE_NULL, "[RunGraph] graph not exist, graph_id=" + std::to_string(args.graph_id)); return; } // more than one graph owns same graph_id uint32_t count = 0; if (graph_manager->GetGraphCount(args.graph_id, count) != SUCCESS) { - GELOGE(INTERNAL_ERROR, "Get graph [id:%u] count failed.", args.graph_id); + GELOGE(INTERNAL_ERROR, "[Get][GraphCount] failed, graph id:%u.", args.graph_id); return; } // Avoid repeatively prerun for graphs owns same graph_id in online inference concurrency @@ -3146,7 +3144,7 @@ void GraphManager::PreRunThread(GraphManager *graph_manager) { graph_node->Lock(); if (graph_node->GetRunFlag()) { - ReturnError(graph_manager, args.callback, GE_GRAPH_GRAPH_NODE_NULL, + ReturnError(graph_manager, args.callback, GE_GRAPH_ALREADY_RUNNING, "[RunGraph] graph already running, graph id=" + std::to_string(args.graph_id)); graph_node->Unlock(); return; @@ -3179,12 +3177,11 @@ void GraphManager::PreRunThread(GraphManager *graph_manager) { if (ret != SUCCESS) { graph_node->SetRunFlag(false); if (!ge::Analyzer::GetInstance()->IsEnableNetAnalyzeDebug()) { - ReturnError(graph_manager, args.callback, ret, "CheckIncreBuildAndPreRun Failed, thread exit.."); + GELOGE(ret, "CheckIncreBuildAndPreRun Failed, thread exit.."); graph_node->Unlock(); return; } else { - ReturnError(graph_manager, graph_node, args.callback, ret, - "CheckIncreBuildAndPreRun Failed, keep geop continue!"); + GELOGE(ret, "CheckIncreBuildAndPreRun Failed, keep geop continue!"); graph_node->Unlock(); continue; } @@ -3218,7 +3215,8 @@ Status GraphManager::ParseInputsDimsForGetNexNosinkAndData(const vector if (!(AttrUtils::GetInt(op_desc, ATTR_NAME_INDEX, index))) { REPORT_CALL_ERROR("E19999", "Get Attr:%s from op:%s(%s) fail", ATTR_NAME_INDEX.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(PARAM_INVALID, "Get index from attr failed"); + GELOGE(PARAM_INVALID, "[Get][Attr] %s from op:%s(%s) fail", ATTR_NAME_INDEX.c_str(), + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return PARAM_INVALID; } if (static_cast(index) > input_tensor.size()) { @@ -3226,7 +3224,9 @@ Status GraphManager::ParseInputsDimsForGetNexNosinkAndData(const vector "check invalid", ATTR_NAME_INDEX.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str(), index, input_tensor.size()); - GELOGE(PARAM_INVALID, "The count of input tensor should be equal to the count of data."); + GELOGE(PARAM_INVALID, "[Check][Param] Attr:%s in op:%s(%s) value:%ld > param input_tensor.size:%zu", + ATTR_NAME_INDEX.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str(), + index, input_tensor.size()); return PARAM_INVALID; } @@ -3256,7 +3256,7 @@ Status GraphManager::ParseInputsDims(const std::vector &input_tensor } else { // data+getnext_nosink, but only need to get shape_dims of data if (ParseInputsDimsForGetNexNosinkAndData(data_nodes, input_tensor) != SUCCESS) { - GELOGE(PARAM_INVALID, "Failed to parse dims from data, when data coexist with getnext nosink."); + GELOGE(PARAM_INVALID, "[Parse][Dims] from data failed, when data coexist with getnext nosink."); return PARAM_INVALID; } } @@ -3272,7 +3272,7 @@ Status GraphManager::ParseInputsDims(const std::vector &input_tensor } else { // getnext_nosink + data, but only need to get shape_dims of getnext_nosink if (ParseInputsDimsForGetNexNosinkAndData(getnext_nosink_nodes, input_tensor) != SUCCESS) { - GELOGE(PARAM_INVALID, "Failed to parse dims from getnext nosink, when data coexist with getnext nosink"); + GELOGE(PARAM_INVALID, "[Parse][Dims] from getnext nosink failed, when data coexist with getnext nosink"); return PARAM_INVALID; } } @@ -3379,9 +3379,8 @@ void GraphManager::ReturnError(GraphManager *graph_manager, GraphNodePtr &graph_ std::vector outputs; auto compute_graph = GraphUtils::GetComputeGraph(*graph_node->GetGraph()); if (graph_manager == nullptr || compute_graph == nullptr) { - REPORT_INNER_ERROR("E19999", "Param graph_manager or compute_graph in graph_node is nullptr, " - "check invalid"); - GELOGE(GRAPH_FAILED, "[Analyze Mode] compute graph is null!"); + REPORT_INNER_ERROR("E19999", "Param graph_manager or compute_graph in graph_node is nullptr, check invalid"); + GELOGE(GRAPH_FAILED, "[Check][Param] compute graph or graph manager is nullptr"); callback(GRAPH_FAILED, outputs); return; } @@ -3404,7 +3403,8 @@ void GraphManager::ReturnError(GraphManager *graph_manager, GraphNodePtr &graph_ REPORT_INNER_ERROR("E19999", "InputIndex:%zu ShapeSize:%ld of op:%s(%s) < 0, unknown shape is not support, " "check invalid", i, len, node->GetName().c_str(), node->GetType().c_str()); - GELOGE(GRAPH_FAILED, "Analyze Mode does not support GEOP output unknown shape!"); + GELOGE(GRAPH_FAILED, "[Check][Param] InputIndex:%zu ShapeSize:%ld of op:%s(%s) < 0, " + "unknown shape is not support", i, len, node->GetName().c_str(), node->GetType().c_str()); callback(GRAPH_FAILED, outputs); return; } else if (len == 0) { @@ -3414,8 +3414,8 @@ void GraphManager::ReturnError(GraphManager *graph_manager, GraphNodePtr &graph_ auto length = GetSizeInBytes(len, input_desc->GetDataType()); auto aligned_ptr = MakeShared(length, kAlignment); if (aligned_ptr == nullptr) { - REPORT_INNER_ERROR("E19999", "Aligned_ptr is nullptr"); - GELOGE(GRAPH_FAILED, "[Analyze Mode] Aligned_ptr is nullptr"); + REPORT_CALL_ERROR("E19999", "New AlignedPtr failed, len:%ld", length); + GELOGE(GRAPH_FAILED, "[Create][AlignedPtr] failed, len:%ld", length); return; } ge_tensor.SetData(aligned_ptr, length); @@ -3437,16 +3437,14 @@ bool GraphManager::IsGraphNeedRebuild(uint32_t graph_id) { GraphNodePtr graph_node = nullptr; Status ret = GetGraphNode(graph_id, graph_node); if (ret != SUCCESS) { - REPORT_INNER_ERROR("E19999", "Graph:%u not exist in graph_map, check invalid", - graph_id); - GELOGE(ret, "[RunGraph] graph not exist, graph_id=%u.", graph_id); + REPORT_INNER_ERROR("E19999", "Graph:%u not exist in graph_map, check invalid", graph_id); + GELOGE(ret, "[Get][GraphNode] failed, graph not exist, graph_id:%u.", graph_id); return true; } if (graph_node == nullptr) { - REPORT_INNER_ERROR("E19999", "Graph node is nullptr in graph_map, graph_id:%u, check invalid", - graph_id); - GELOGE(GE_GRAPH_GRAPH_NODE_NULL, "[RunGraph] graph node is NULL, graphId=%u.", graph_id); + REPORT_INNER_ERROR("E19999", "Graph node is nullptr in graph_map, graph_id:%u, check invalid", graph_id); + GELOGE(GE_GRAPH_GRAPH_NODE_NULL, "[Check][Param] graph node is NULL, graph_id:%u.", graph_id); return true; } @@ -3460,16 +3458,14 @@ const map *GraphManager::GetGraphOptions(uint32_t grap GraphNodePtr graph_node = nullptr; Status ret = GetGraphNode(graph_id, graph_node); if (ret != SUCCESS) { - REPORT_INNER_ERROR("E19999", "Graph:%u not exist in graph_map, check invalid", - graph_id); - GELOGE(ret, "[RunGraph] graph not exist, graph_id=%u.", graph_id); + REPORT_INNER_ERROR("E19999", "Graph:%u not exist in graph_map, check invalid", graph_id); + GELOGE(ret, "[Get][GraphNode] failed, graph not exist, graph_id:%u.", graph_id); return nullptr; } if (!graph_node) { - REPORT_INNER_ERROR("E19999", "Graph node is nullptr in graph_map, graph_id:%u, check invalid", - graph_id); - GELOGE(GE_GRAPH_GRAPH_NODE_NULL, "[RunGraph] graph node is NULL, graph_id=%u.", graph_id); + REPORT_INNER_ERROR("E19999", "Graph node is nullptr in graph_map, graph_id:%u, check invalid", graph_id); + GELOGE(GE_GRAPH_GRAPH_NODE_NULL, "[Check][Param] graph node is NULL, graph_id:%u.", graph_id); return nullptr; } return &(graph_node->GetOptions()); @@ -3485,7 +3481,7 @@ Status GraphManager::OptimizeSubgraph(const GraphNodePtr &graph_node, ComputeGra StagePartitioner stage_partitioner(compute_graph); auto ret = stage_partitioner.Partition(); if (ret != SUCCESS) { - GELOGE(ret, "Graph partition by stage Failed"); + GELOGE(ret, "[Call][Partition] for Graph:%s by stage Failed", compute_graph->GetName().c_str()); return ret; } GE_TIMESTAMP_EVENT_END(StagePartition, "OptimizeSubgraph::StagePartition"); @@ -3494,14 +3490,16 @@ Status GraphManager::OptimizeSubgraph(const GraphNodePtr &graph_node, ComputeGra DynamicShapePartitioner dynamic_shape_partitioner(compute_graph); ret = dynamic_shape_partitioner.Partition(); if (ret != SUCCESS) { - GELOGE(ret, "Graph partition by dynamic shape Failed"); + GELOGE(ret, "[Call][Partition] for Graph:%s by dynamic shape Failed", compute_graph->GetName().c_str()); return ret; } bool dynamic_shape_partitioned = false; if (!AttrUtils::GetBool(*compute_graph, ATTR_NAME_DYNAMIC_SHAPE_PARTITIONED, dynamic_shape_partitioned)) { - REPORT_INNER_ERROR("E19999", "Get Attr:%s from graph:%u fail", - ATTR_NAME_DYNAMIC_SHAPE_PARTITIONED.c_str(), compute_graph->GetGraphID()); - GELOGE(FAILED, "failed get dynamic shape partitioned flag on partitioned graph."); + REPORT_INNER_ERROR("E19999", "Get Attr:%s from graph:%s(id:%u) fail", + ATTR_NAME_DYNAMIC_SHAPE_PARTITIONED.c_str(), compute_graph->GetName().c_str(), + compute_graph->GetGraphID()); + GELOGE(FAILED, "[Get][Attr] %s from graph:%u failed", + ATTR_NAME_DYNAMIC_SHAPE_PARTITIONED.c_str(), compute_graph->GetGraphID()); return FAILED; } GE_TIMESTAMP_EVENT_END(GraphPartitionDynamicShape, "OptimizeSubgraph::GraphPartitionDynamicShape"); @@ -3510,14 +3508,14 @@ Status GraphManager::OptimizeSubgraph(const GraphNodePtr &graph_node, ComputeGra GraphPartitioner &partitioner = GetCompilerStages(graph_node->GetGraphId()).partitioner; ret = partitioner.Partition(compute_graph, GraphPartitioner::kPartitioning); if (ret != SUCCESS) { - GELOGE(ret, "Graph partition Failed"); + GELOGE(ret, "[Call][Partition] for Graph:%s Failed", compute_graph->GetName().c_str()); return ret; } GE_TIMESTAMP_EVENT_END(GraphPartition, "OptimizeSubgraph::Partition1"); GE_TIMESTAMP_START(SetSubgraph); ret = SetSubgraph(session_id, compute_graph, partitioner); if (ret != SUCCESS) { - GELOGE(ret, "Graph set subgraph Failed"); + GELOGE(ret, "[Set][Subgraph] failed for graph:%s, session_id:%lu", compute_graph->GetName().c_str(), session_id); return ret; } GE_TIMESTAMP_EVENT_END(SetSubgraph, "OptimizeSubgraph::SetSubGraph"); @@ -3530,7 +3528,7 @@ Status GraphManager::OptimizeSubgraph(const GraphNodePtr &graph_node, ComputeGra Status ret = ConvertGraphToFile(compute_graph, partitioner, tuning_path, (options_.build_step == BUILD_STEP_AFTER_BUILDER)); if (ret != SUCCESS) { - GELOGE(ret, "Convert graph[%s] to file failed", compute_graph->GetName().c_str()); + GELOGE(ret, "[Convert][Graph] [%s] to file failed", compute_graph->GetName().c_str()); return ret; } GE_TIMESTAMP_EVENT_END(ConvertGraphToFile, "OptimizeSubgraph::ConvertGraphToFile"); @@ -3543,7 +3541,8 @@ Status GraphManager::OptimizeSubgraph(const GraphNodePtr &graph_node, ComputeGra GE_TIMESTAMP_START(MergeSubgraph); ret = MergeSubGraph(merged_compute_graph, compute_graph, graph_node->GetGraphId()); if (ret != SUCCESS) { - GELOGE(ret, "Merge SubGraph Failed"); + GELOGE(ret, "[Merge][SubGraph] Failed, graph:%s(id:%u)", + compute_graph->GetName().c_str(), graph_node->GetGraphId()); return ret; } GE_CHECK_NOTNULL(merged_compute_graph); @@ -3560,8 +3559,8 @@ Status GraphManager::OptimizeSubgraph(const GraphNodePtr &graph_node, ComputeGra if (!AttrUtils::SetBool(merged_compute_graph, ATTR_NAME_OFF_SUPERKERNEL_ATTR, off_superkernel)) { REPORT_INNER_ERROR("E19999", "Set Attr:%s to graph:%u fail", ATTR_NAME_OFF_SUPERKERNEL_ATTR.c_str(), compute_graph->GetGraphID()); - GELOGE(FAILED, "Compute graph %s set superkernel flag %d failed", merged_compute_graph->GetName().c_str(), - off_superkernel); + GELOGE(FAILED, "[Set][Attr] %s to graph:%u fail", + ATTR_NAME_OFF_SUPERKERNEL_ATTR.c_str(), compute_graph->GetGraphID()); return FAILED; } } @@ -3571,7 +3570,8 @@ Status GraphManager::OptimizeSubgraph(const GraphNodePtr &graph_node, ComputeGra if (!AttrUtils::SetBool(*compute_graph, ATTR_NAME_DYNAMIC_SHAPE_PARTITIONED, dynamic_shape_partitioned)) { REPORT_INNER_ERROR("E19999", "Set Attr:%s to graph:%u fail", ATTR_NAME_DYNAMIC_SHAPE_PARTITIONED.c_str(), compute_graph->GetGraphID()); - GELOGE(FAILED, "failed set dynamic shape partitioned flag on partitioned graph."); + GELOGE(FAILED, "[Set][Attr] %s to graph:%u fail", + ATTR_NAME_DYNAMIC_SHAPE_PARTITIONED.c_str(), compute_graph->GetGraphID()); return FAILED; } return SUCCESS; @@ -3631,7 +3631,7 @@ Status GraphManager::Build(const GraphNodePtr &graph_node, ComputeGraphPtr &comp auto ret = GetCompilerStages(graph_node->GetGraphId()).builder.Build(compute_graph, ge_root_model, session_id); if (ret != SUCCESS) { - GELOGE(ret, "SubGraph build Failed."); + GELOGE(ret, "[Call][Build] failed, session_id:%lu.", session_id); return ret; } @@ -3669,7 +3669,7 @@ Status GraphManager::GenCheckPointGraph(const std::mapAddOutputDesc(iter->second)); NodePtr var_node = compute_graph->AddNode(var_desc); GE_CHK_STATUS(GraphUtils::AddEdge(var_node->GetOutDataAnchor(0), save_node->GetInDataAnchor(index)), - "Add edge[%s->%s] fail.", var_node->GetName().c_str(), save_node->GetName().c_str()); + "[Add][Edge][%s->%s] fail.", var_node->GetName().c_str(), save_node->GetName().c_str()); index++; } compute_graph->Dump(); @@ -3680,13 +3680,12 @@ Status GraphManager::GenCheckPointGraph(const std::map &var_names, const std::vector &outputs, std::vector &var_values) { map var_results; - GE_CHK_STATUS_RET(SaveCheckPointResult(graph, outputs, var_results), "Save check point result failed."); + GE_CHK_STATUS_RET(SaveCheckPointResult(graph, outputs, var_results), "[Save][CheckPointResult] failed."); if (!var_names.empty()) { for (const auto &var_name : var_names) { if (var_results.count(var_name) == 0) { - REPORT_INNER_ERROR("E19999", "Fetch Var:%s result value fail", - var_name.c_str()); - GELOGE(FAILED, "Fetch var[%s] value failed.", var_name.c_str()); + REPORT_INNER_ERROR("E19999", "Fetch Var:%s result value fail", var_name.c_str()); + GELOGE(FAILED, "[Check][Param] Fetch var[%s] value failed.", var_name.c_str()); return FAILED; } else { auto var_tensor = var_results[var_name].GetTensorDesc(); @@ -3727,7 +3726,8 @@ Status GraphManager::SaveCheckPointResult(const Graph &graph, const std::vector< REPORT_INNER_ERROR("E19999", "peer node:%s(%s) of netoutput has more than 1 input in checkpoint Graph, " "check invalid", peer_node->GetName().c_str(), peer_node->GetType().c_str()); - GELOGE(FAILED, "peer_node [%s] has more than 1 input in checkpoint Graph.", peer_node->GetName().c_str()); + GELOGE(FAILED, "[Check][Param] peer_node [%s] has more than 1 input in checkpoint Graph.", + peer_node->GetName().c_str()); return FAILED; } auto peer_node_in_anchor = peer_node->GetAllInDataAnchors().at(0); @@ -3743,7 +3743,7 @@ Status GraphManager::SaveCheckPointResult(const Graph &graph, const std::vector< REPORT_INNER_ERROR("E19999", "peer node:%s(%s) of netoutput is not variable in checkpoint Graph, " "check invalid", peer_node->GetName().c_str(), peer_node->GetType().c_str()); - GELOGE(FAILED, " peer_node %s is not variable in checkpoint Graph.", peer_node->GetName().c_str()); + GELOGE(FAILED, "[Check][Param] peer_node %s is not variable in checkpoint Graph.", peer_node->GetName().c_str()); return FAILED; } auto var_name = peer_node->GetName(); @@ -3751,7 +3751,7 @@ Status GraphManager::SaveCheckPointResult(const Graph &graph, const std::vector< if (in->GetIdx() >= static_cast(outputs.size())) { REPORT_INNER_ERROR("E19999", "In index:%u of netoutput is out of outputs.size:%zu range in checkpoint Graph, " "check invalid", in->GetIdx(), outputs.size()); - GELOGE(FAILED, "variable index[%d] out of range[%zu].", in->GetIdx(), outputs.size()); + GELOGE(FAILED, "[Check][Param] variable index[%d] out of range[%zu].", in->GetIdx(), outputs.size()); return FAILED; } var_results.emplace(var_name, outputs.at(in->GetIdx())); diff --git a/ge/graph/manager/graph_manager_utils.cc b/ge/graph/manager/graph_manager_utils.cc index 0f93654c..a70b15a6 100644 --- a/ge/graph/manager/graph_manager_utils.cc +++ b/ge/graph/manager/graph_manager_utils.cc @@ -46,7 +46,7 @@ GraphNode::GraphNode(GraphId graph_id) sem_(1) { graph_run_async_listener_ = MakeShared(); if (graph_run_async_listener_ == nullptr) { - GELOGE(MEMALLOC_FAILED, "Make shared failed"); + GELOGE(MEMALLOC_FAILED, "[New][RunAsyncListener] failed"); } } @@ -82,7 +82,8 @@ SubGraphInfo::~SubGraphInfo() { rt_ret = rtFreeHost(buffer_addr); buffer_addr = nullptr; if (rt_ret != RT_ERROR_NONE) { - GELOGE(rt_ret, "[GraphManager] subgraph free buffer failed, modelId = %u", model_id_info_.model_id); + GELOGE(rt_ret, "[Call][RtFreeHost] subgraph free buffer failed, modelId = %u", + model_id_info_.model_id); } } } @@ -94,8 +95,8 @@ Status SubGraphInfo::FreeInOutBuffer() { rtError_t rt_ret; rt_ret = rtFreeHost(*iter); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtFreeHost fail"); - GELOGE(rt_ret, "[GraphManager] subgraph free buffer failed, modelId = %u", model_id_info_.model_id); + REPORT_CALL_ERROR("E19999", "Call rtFreeHost fail, ret:%d", rt_ret); + GELOGE(rt_ret, "[Call][RtFreeHost] subgraph free buffer failed, modelId = %u", model_id_info_.model_id); buffer_addr_.erase(buffer_addr_.begin(), iter); return GE_GRAPH_FREE_FAILED; } @@ -131,7 +132,7 @@ Status GraphModelListener::OnComputeDone(uint32_t model_id, uint32_t task_id, ui uint32_t GraphModelListener::GetResultCode() const { if (!is_finished_) { REPORT_CALL_ERROR("E19999", "Model not run finish"); - GELOGE(INTERNAL_ERROR, "[GraphManager] model not run finish."); + GELOGE(INTERNAL_ERROR, "[Check][Param] model not run finish."); return INTERNAL_ERROR; } return result_code_; @@ -170,7 +171,9 @@ bool HasCalcOp(const ComputeGraphPtr &graph) { for (const auto &node : graph->GetAllNodes()) { OpDescPtr op_desc = node->GetOpDesc(); - GE_IF_BOOL_EXEC(op_desc == nullptr, GELOGE(FAILED, "Node GetOpDesc is nullptr"); return false); + GE_IF_BOOL_EXEC(op_desc == nullptr, + REPORT_INNER_ERROR("E19999", "GetOpDesc failed, Node GetOpDesc is nullptr"); + GELOGE(FAILED, "[Get][OpDesc] failed, Node GetOpDesc is nullptr"); return false); if (calc_op_type.find(op_desc->GetType()) != calc_op_type.end()) { return true; } diff --git a/ge/graph/manager/graph_mem_allocator.cc b/ge/graph/manager/graph_mem_allocator.cc index 0cccaf99..dd38274e 100755 --- a/ge/graph/manager/graph_mem_allocator.cc +++ b/ge/graph/manager/graph_mem_allocator.cc @@ -50,9 +50,7 @@ uint8_t *MemoryAllocator::MallocMemory(const string &purpose, size_t memory_size if (rtMalloc(reinterpret_cast(&memory_addr), memory_size, memory_type_) != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtMalloc fail, purpose:%s, size:%zu, device_id:%u", purpose.c_str(), memory_size, device_id); - GELOGE(ge::INTERNAL_ERROR, - "MemoryAllocator::MallocMemory device_id = %u," - " size= %lu", + GELOGE(ge::INTERNAL_ERROR, "[Malloc][Memory] failed, device_id = %u, size= %lu", device_id, memory_size); return nullptr; @@ -68,7 +66,7 @@ Status MemoryAllocator::FreeMemory(uint8_t *memory_addr, uint32_t device_id) con auto rtRet = rtFree(memory_addr); if (rtRet != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtFree fail, device_id:%u", device_id); - GELOGE(rtRet, "MemoryAllocator::MallocMemory device_id = %u", device_id); + GELOGE(rtRet, "[Call][RtFree] failed, device_id = %u", device_id); return RT_ERROR_TO_GE_STATUS(rtRet); } memory_addr = nullptr; @@ -88,10 +86,8 @@ uint8_t *MemoryAllocator::MallocMemory(const string &purpose, const string &memo if (memory_addr == nullptr) { REPORT_CALL_ERROR("E19999", "Malloc Memory fail, purpose:%s, memory_key:%s, memory_size:%zu, device_id:%u", purpose.c_str(), memory_key.c_str(), memory_size, device_id); - GELOGE(ge::INTERNAL_ERROR, - "MemoryAllocator::MallocMemory failed," - " memory_key[%s], size = %lu.", - memory_key.c_str(), memory_size); + GELOGE(ge::INTERNAL_ERROR, "[Malloc][Memory] failed, memory_key[%s], size = %lu, device_id:%u.", + memory_key.c_str(), memory_size, device_id); return nullptr; } @@ -126,10 +122,8 @@ Status MemoryAllocator::FreeMemory(const string &memory_key, uint32_t device_id) if (FreeMemory(it->second.memory_addr_, device_id) != ge::SUCCESS) { REPORT_CALL_ERROR("E19999", "Free Memory fail, memory_key:%s, device_id:%u", memory_key.c_str(), device_id); - GELOGE(ge::INTERNAL_ERROR, - "MemoryAllocator::FreeMemory rtFree failed," - " memory_key[%s]", - memory_key.c_str()); + GELOGE(ge::INTERNAL_ERROR, "[Free][Memory] failed, memory_key[%s], device_id:%u", + memory_key.c_str(), device_id); return ge::INTERNAL_ERROR; } diff --git a/ge/graph/manager/graph_var_manager.cc b/ge/graph/manager/graph_var_manager.cc index 5f7586da..ced8465f 100755 --- a/ge/graph/manager/graph_var_manager.cc +++ b/ge/graph/manager/graph_var_manager.cc @@ -40,7 +40,8 @@ ge::Status VarResource::GetVarAddr(const std::string &var_name, const ge::GeTens if (dev_ptr == nullptr) { REPORT_INNER_ERROR("E19999", "Param dev_ptr is nullptr, var_name:%s, session_id:%lu, " "check invalid", var_name.c_str(), session_id_); - GELOGE(FAILED, "[GetVarAddr] dev_ptr is null!"); + GELOGE(FAILED, "[Check][Param] Param dev_ptr is nullptr, var_name:%s, session_id:%lu", + var_name.c_str(), session_id_); return FAILED; } std::string var_key = VarKey(var_name, tensor_desc); @@ -51,7 +52,8 @@ ge::Status VarResource::GetVarAddr(const std::string &var_name, const ge::GeTens REPORT_INNER_ERROR("E19999", "var_key:%s can't find in var_addr_mgr_map_, var_name:%s, session_id:%lu, " "check invalid", var_key.c_str(), var_name.c_str(), session_id_); - GELOGE(FAILED, "VarResource::GetVarAddr failed, var_key %s", var_key.c_str()); + GELOGE(FAILED, "[Check][Param] var_key:%s can't find in var_addr_mgr_map_, var_name:%s, session_id:%lu", + var_key.c_str(), var_name.c_str(), session_id_); return FAILED; } @@ -109,7 +111,8 @@ ge::Status VarResource::SaveVarAddr(const std::string &var_name, const ge::GeTen REPORT_INNER_ERROR("E19999", "var_key:%s conflict in var_addr_mgr_map_, var_name:%s, session_id:%lu, " "check invalid", var_key.c_str(), var_name.c_str(), session_id_); - GELOGE(FAILED, "VarResource::SaveVarAddr, var_key %s save addr conflict", var_key.c_str()); + GELOGE(FAILED, "[Check][Param] var_key:%s conflict in var_addr_mgr_map_, var_name:%s, session_id:%lu", + var_key.c_str(), var_name.c_str(), session_id_); return FAILED; } @@ -145,14 +148,15 @@ ge::Status VarResource::RenewCurVarDesc(const std::string &var_name, const ge::O if (op_desc == nullptr) { REPORT_INNER_ERROR("E19999", "Param op_desc is nullptr, var_name:%s, session_id:%lu, check invalid", var_name.c_str(), session_id_); - GELOGE(FAILED, "[RenewCurVarDesc] renew var desc fail! input opdesc is null!"); + GELOGE(FAILED, "[Check][Param] input opdesc is nullptr, var_name:%s, session_id:%lu", + var_name.c_str(), session_id_); return FAILED; } ge::GeTensorDesc curr_desc; ge::Status ret = GetCurVarDesc(var_name, curr_desc); if (ret != SUCCESS) { - GELOGE(FAILED, "[RenewCurVarDesc] Get var desc fail!"); + GELOGE(FAILED, "[Get][CurVarDesc] fail, var_name:%s, session_id:%lu", var_name.c_str(), session_id_); return FAILED; } std::string key = VarKey(var_name, curr_desc); @@ -164,7 +168,8 @@ ge::Status VarResource::RenewCurVarDesc(const std::string &var_name, const ge::O REPORT_INNER_ERROR("E19999", "var_key:%s can't find in var_addr_mgr_map_, var_name:%s, session_id:%lu, op:%s(%s), " "check invalid", key.c_str(), var_name.c_str(), session_id_, op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(FAILED, "[RenewCurVarDesc] can't find ele with key [%s]", key.c_str()); + GELOGE(FAILED, "[Check][Param] var_key:%s can't find in var_addr_mgr_map_, var_name:%s, session_id:%lu, op:%s(%s)", + key.c_str(), var_name.c_str(), session_id_, op_desc->GetName().c_str(), op_desc->GetType().c_str()); return FAILED; } auto val = iter->second; @@ -285,14 +290,15 @@ Status HbmMemResource::AssignVarMem(const std::string &var_name, uint64_t size, if (total_size_ < var_mem_size_) { REPORT_INNER_ERROR("E19999", "VarMemMaxSize:%lu < var_mem_size_:%lu, var_size:%lu, var_name:%s, check invalid" "", total_size_, var_mem_size_, size, var_name.c_str()); - GELOGE(PARAM_INVALID, "total_size_: %lu is smaller than var_mem_size_: %lu", total_size_, var_mem_size_); + GELOGE(PARAM_INVALID, "[Check][Param] total_size_:%lu is smaller than var_mem_size_:%lu, var_name:%s", + total_size_, var_mem_size_, var_name.c_str()); return PARAM_INVALID; } uint64_t free_size = total_size_ - var_mem_size_; if (free_size < (size + kSessionMemAlignSize * kSessionMemAlignUnit)) { REPORT_INNER_ERROR("E19999", "free_size:%lu not enough, var_align_size:%lu, var_name:%s, check invalid", free_size, size, var_name.c_str()); - GELOGE(PARAM_INVALID, "Out of memory : current var size[%lu] exceeds total var size[%lu]", + GELOGE(PARAM_INVALID, "[Check][Param] Out of memory: current var size[%lu] exceeds total var size[%lu]", size + kSessionMemAlignSize * kSessionMemAlignUnit + var_mem_size_, total_size_); return PARAM_INVALID; } @@ -317,7 +323,7 @@ Status RdmaMemResource::AssignVarMem(const std::string &var_name, uint64_t size, if (buffer == nullptr) { REPORT_CALL_ERROR("E19999", "malloc rdma memory fail, var_size:%lu, var_name:%s", size, var_name.c_str()); - GELOGE(MEMALLOC_FAILED, "Failed to malloc rdma memory for node %s, size = %lu", var_name.c_str(), size); + GELOGE(MEMALLOC_FAILED, "[Malloc][RdmaMemory] for node %s failed, size = %lu", var_name.c_str(), size); return MEMALLOC_FAILED; } address = static_cast(reinterpret_cast(buffer)); @@ -468,7 +474,8 @@ int64_t VarManager::GetVarMemSize(rtMemType_t memory_type) { if (mem_resource == nullptr) { REPORT_INNER_ERROR("E19999", "Find no mem_resource in map, memory_type:%d, session_id:%lu", memory_type, session_id_); - GELOGE(ge::INTERNAL_ERROR, "MemResource is invalid."); + GELOGE(ge::INTERNAL_ERROR, "[Check][Param] MemResource is invalid, memory_type:%d, session_id:%lu", + memory_type, session_id_); return 0; } return mem_resource->GetVarMemSize(); @@ -482,8 +489,9 @@ Status VarManager::UpdateVarMemSize(rtMemType_t memory_type, int64_t mem_size) { mem_resource = MemResource::BuildMemResourceFromType(memory_type); if (mem_resource == nullptr) { REPORT_CALL_ERROR("E19999", "memory_type:%d invalid or New MemResource fail, session_id:%lu", - memory_type, session_id_); - GELOGE(ge::INTERNAL_ERROR, "Alloc MemResource failed, memory_type = %u.", memory_type); + memory_type, session_id_); + GELOGE(ge::INTERNAL_ERROR, "[Alloc][MemResource] failed, memory_type:%u, session_id:%lu", + memory_type, session_id_); return ge::INTERNAL_ERROR; } else { mem_resource_map_[memory_type] = mem_resource; @@ -495,7 +503,8 @@ Status VarManager::UpdateVarMemSize(rtMemType_t memory_type, int64_t mem_size) { if (mem_resource == nullptr) { REPORT_INNER_ERROR("E19999", "MemResource is invalid, memory_type:%d, session_id:%lu", memory_type, session_id_); - GELOGE(ge::INTERNAL_ERROR, "MemResource is invalid."); + GELOGE(ge::INTERNAL_ERROR, "[Check][Param] MemResource is invalid, memory_type:%u, session_id:%lu", + memory_type, session_id_); return FAILED; } mem_resource->UpdateVarMemSize(mem_size); @@ -515,7 +524,8 @@ ge::Status VarManager::AssignVarMem(const std::string &var_name, const ge::GeTen if (result != ge::SUCCESS) { REPORT_CALL_ERROR("E19999", "Get size from tensor fail, var_name:%s, memory_type:%d, session_id:%lu", var_name.c_str(), memory_type, session_id_); - GELOGE(result, "get size from TensorDesc failed"); + GELOGE(result, "[Get][Size] from tensor fail, var_name:%s, memory_type:%u, session_id:%lu", + var_name.c_str(), memory_type, session_id_); return result; } @@ -526,7 +536,8 @@ ge::Status VarManager::AssignVarMem(const std::string &var_name, const ge::GeTen if (mem_resource == nullptr) { REPORT_CALL_ERROR("E19999", "memory_type:%d invalid or New MemResource fail, session_id:%lu", memory_type, session_id_); - GELOGE(ge::INTERNAL_ERROR, "Alloc MemResource failed, memory_type = %u.", memory_type); + GELOGE(ge::INTERNAL_ERROR, "[Alloc][MemResource] failed, memory_type:%u, session_id:%lu.", + memory_type, session_id_); return ge::INTERNAL_ERROR; } else { mem_resource_map_[memory_type] = mem_resource; @@ -538,7 +549,8 @@ ge::Status VarManager::AssignVarMem(const std::string &var_name, const ge::GeTen if (mem_resource == nullptr) { REPORT_INNER_ERROR("E19999", "MemResource is invalid, memory_type:%d, session_id:%lu", memory_type, session_id_); - GELOGE(ge::INTERNAL_ERROR, "MemResource is invalid, memory_type = %u.", memory_type); + GELOGE(ge::INTERNAL_ERROR, "[Check][Param] MemResource is invalid, memory_type:%u, session_id:%lu.", + memory_type, session_id_); return ge::INTERNAL_ERROR; } @@ -567,14 +579,15 @@ ge::Status VarManager::AssignVarMem(const std::string &var_name, const ge::GeTen if (can_not_reuse_old_memory) { result = mem_resource->AssignVarMem(var_name, tensor_desc_size, session_id_, mem_offset); if (result != SUCCESS) { - GELOGE(ge::INTERNAL_ERROR, "AssignVarMem by offset failed."); + GELOGE(ge::INTERNAL_ERROR, "[Assign][VarMem] by offset failed, session_id:%lu.", session_id_); return ge::INTERNAL_ERROR; } result = var_resource_->SaveVarAddr( var_name, tensor_desc, reinterpret_cast(static_cast(mem_offset)), memory_type); if (result != SUCCESS) { - GELOGE(ge::INTERNAL_ERROR, "AssignVarMem by offset failed."); + GELOGE(ge::INTERNAL_ERROR, "[Save][VarAddr] by offset failed, memory type:%u, session_id:%lu.", + memory_type, session_id_); return ge::INTERNAL_ERROR; } } @@ -681,7 +694,8 @@ ge::Status VarManager::RenewCurVarDesc(const std::string &var_name, ge::OpDescPt REPORT_INNER_ERROR("E19999", "VarManager has not been init, op:%s(%s), session_id:%lu, check invalid", op_desc->GetName().c_str(), op_desc->GetType().c_str(), session_id_); - GELOGE(ge::INTERNAL_ERROR, "VarManager has not been init."); + GELOGE(ge::INTERNAL_ERROR, "[Check][Param] VarManager has not been init, op:%s(%s), session_id:%lu", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), session_id_); return ge::INTERNAL_ERROR; } return var_resource_->RenewCurVarDesc(var_name, std::move(op_desc)); @@ -729,10 +743,8 @@ ge::Status VarManager::MallocVarMemory(size_t memory_size) { const string purpose("variables and constant op memory in training network."); var_mem_base = MemManager::Instance().MemInstance(RT_MEMORY_HBM).MallocMemory(purpose, memory_key, var_memory_size); if (var_mem_base == nullptr) { - GELOGE(ge::INTERNAL_ERROR, - "VarManager::MallocVarMemory failed " - "session_id = %s", - memory_key.c_str()); + GELOGE(ge::INTERNAL_ERROR, "[Malloc][VarMemory] failed, size:%zu, session_id:%s", + var_memory_size, memory_key.c_str()); return ge::INTERNAL_ERROR; } return SUCCESS; @@ -812,7 +824,7 @@ Status VarManager::SetMemoryMallocSize(const map &options) { string graph_memory_manager_malloc_max_size = it->second; ge::Status ret = ParseMemoryMallocSize(graph_memory_manager_malloc_max_size, graph_mem_max_size_); if (ret != SUCCESS) { - GELOGE(ge::GE_GRAPH_OPTIONS_INVALID, "Parse graph memory manager malloc max size failed."); + GELOGE(ge::GE_GRAPH_OPTIONS_INVALID, "[Call][ParseMemoryMallocSize] failed, session id:%lu.", session_id_); return ge::GE_GRAPH_OPTIONS_INVALID; } GELOGI("The max size for graph mem is set to %zu", graph_mem_max_size_); @@ -825,7 +837,7 @@ Status VarManager::SetMemoryMallocSize(const map &options) { string memory_var_manager_malloc_size = it->second; ge::Status ret = ParseMemoryMallocSize(memory_var_manager_malloc_size, var_mem_max_size_); if (ret != SUCCESS) { - GELOGE(ge::GE_GRAPH_OPTIONS_INVALID, "Parse memory var manager malloc size failed."); + GELOGE(ge::GE_GRAPH_OPTIONS_INVALID, "[Call][ParseMemoryMallocSize] failed, session id:%lu.", session_id_); return ge::GE_GRAPH_OPTIONS_INVALID; } } @@ -834,8 +846,8 @@ Status VarManager::SetMemoryMallocSize(const map &options) { if (var_mem_logic_base_ > kMaxMemorySize) { REPORT_INNER_ERROR("E19999", "var_login_base:%zu can not exeed limit:%zu, session_id:%lu, check invalid", var_mem_logic_base_, kMaxMemorySize, session_id_); - GELOGE(ge::GE_GRAPH_OPTIONS_INVALID, "kMemoryVarLogicBase : %zu can not exceed max memory size : %zu.", - var_mem_logic_base_, kMaxMemorySize); + GELOGE(ge::GE_GRAPH_OPTIONS_INVALID, "[Check][Param] kMemoryVarLogicBase:%zu can not exceed " + "max memory size:%zu, session_id:%lu.", var_mem_logic_base_, kMaxMemorySize, session_id_); return ge::GE_GRAPH_OPTIONS_INVALID; } @@ -843,8 +855,8 @@ Status VarManager::SetMemoryMallocSize(const map &options) { if (use_max_mem_size_ > kMaxMemorySize) { REPORT_INNER_ERROR("E19999", "all mem_use size:%zu can not exeed limit:%zu, session_id:%lu, check invalid", use_max_mem_size_, kMaxMemorySize, session_id_); - GELOGE(ge::GE_GRAPH_OPTIONS_INVALID, "kUseMaxMemorySize : %zu can not exceed max memory size : %zu.", - use_max_mem_size_, kMaxMemorySize); + GELOGE(ge::GE_GRAPH_OPTIONS_INVALID, "[Check][Param] kUseMaxMemorySize:%zu can not exceed " + "max memory size:%zu, session_id:%lu.", use_max_mem_size_, kMaxMemorySize, session_id_); return ge::GE_GRAPH_OPTIONS_INVALID; } GELOGI("Set memory malloc size successfully"); @@ -855,7 +867,7 @@ Status VarManager::ParseMemoryMallocSize(string &memory_size, size_t &result) { if (memory_size.empty()) { REPORT_INNER_ERROR("E19999", "Param memory_size is empty, session_id:%lu, check invalid", session_id_); - GELOGE(GE_GRAPH_OPTIONS_INVALID, "Memory malloc size input is empty."); + GELOGE(GE_GRAPH_OPTIONS_INVALID, "[Check][Param] Memory malloc size input is empty, session_id:%lu.", session_id_); return GE_GRAPH_OPTIONS_INVALID; } // split string by '*' @@ -882,7 +894,9 @@ Status VarManager::ParseMemoryMallocSize(string &memory_size, size_t &result) { if (!isdigit(c)) { REPORT_INNER_ERROR("E19999", "Param memory_size:%s contains non digit, session_id:%lu, check invalid", memory_size.c_str(), session_id_); - GELOGE(GE_GRAPH_OPTIONS_INVALID, "Memory malloc size input contains non digit."); + GELOGE(GE_GRAPH_OPTIONS_INVALID, + "[Check][Param] Memory malloc size:%s input contains non digit, session_id:%lu.", + memory_size.c_str(), session_id_); return GE_GRAPH_OPTIONS_INVALID; } } @@ -891,13 +905,15 @@ Status VarManager::ParseMemoryMallocSize(string &memory_size, size_t &result) { REPORT_INNER_ERROR("E19999", "Param memory_size:%s will overflow after multi all, session_id:%lu, " "check invalid", memory_size.c_str(), session_id_); - GELOGE(FAILED, "Input memory size is out of range."); + GELOGE(FAILED, "[Check][Param] Param memory_size:%s will overflow after multi all, session_id:%lu", + memory_size.c_str(), session_id_); return FAILED); if ((num > kMaxMemorySize) || (result * static_cast(num) > kMaxMemorySize)) { REPORT_INNER_ERROR("E19999", "Param memory_size:%s after multi will exceed limit:%lu, session_id:%lu, " "check invalid", memory_size.c_str(), kMaxMemorySize, session_id_); - GELOGE(FAILED, "Input memory size can not exceed max memory size : %zu.", kMaxMemorySize); + GELOGE(FAILED, "[Check][Param] Input memory size can not exceed max memory size:%zu, session_id:%lu.", + kMaxMemorySize, session_id_); return FAILED; } result *= static_cast(num); @@ -1001,10 +1017,7 @@ VarManager *VarManagerPool::GetVarManager(uint64_t session_id) { VarManager *var_manager = new (std::nothrow) VarManager(session_id); if (var_manager == nullptr) { REPORT_INNER_ERROR("E19999", "New VarManager fail, session_id:%lu", session_id); - GELOGE(INTERNAL_ERROR, - "VarManager::Instance find session by " - "session_id[%lu] failed.", - session_id); + GELOGE(INTERNAL_ERROR, "[New][VarManager] fail, session_id:%lu", session_id); static VarManager new_var_manager(0); return &new_var_manager; } diff --git a/ge/graph/manager/host_mem_allocator.cc b/ge/graph/manager/host_mem_allocator.cc index 98f9a313..e349719c 100644 --- a/ge/graph/manager/host_mem_allocator.cc +++ b/ge/graph/manager/host_mem_allocator.cc @@ -21,7 +21,10 @@ namespace ge { const void *HostMemAllocator::Malloc(const std::shared_ptr &aligned_ptr, size_t size) { if (aligned_ptr == nullptr) { - GELOGW("Insert a null aligned_ptr"); + GELOGW("Insert a null aligned_ptr, size=%zu", size); + if (size == 0) { + allocated_blocks_[nullptr] = { size, nullptr }; + } return nullptr; } GELOGD("allocate existed host memory succ, size=%zu", size); @@ -34,8 +37,8 @@ uint8_t *HostMemAllocator::Malloc(size_t size) { std::lock_guard lock(mutex_); std::shared_ptr aligned_ptr = MakeShared(size); if (aligned_ptr == nullptr) { - REPORT_INNER_ERROR("E19999", "New AlignedPtr fail"); - GELOGE(INTERNAL_ERROR, "make shared_ptr for AlignedPtr failed"); + REPORT_INNER_ERROR("E19999", "New AlignedPtr fail, size:%zu", size); + GELOGE(INTERNAL_ERROR, "[Call][MakeShared] for AlignedPtr failed, size:%zu", size); return nullptr; } allocated_blocks_[aligned_ptr->Get()] = { size, aligned_ptr }; @@ -46,7 +49,7 @@ uint8_t *HostMemAllocator::Malloc(size_t size) { Status HostMemAllocator::Free(const void *memory_addr) { if (memory_addr == nullptr) { REPORT_INNER_ERROR("E19999", "Param memory_addr is nullptr, check invalid"); - GELOGE(GE_GRAPH_FREE_FAILED, "Invalid memory pointer"); + GELOGE(GE_GRAPH_FREE_FAILED, "[Check][Param] Invalid memory pointer"); return GE_GRAPH_FREE_FAILED; } @@ -54,7 +57,7 @@ Status HostMemAllocator::Free(const void *memory_addr) { auto it = allocated_blocks_.find(memory_addr); if (it == allocated_blocks_.end()) { REPORT_INNER_ERROR("E19999", "Memory_addr is not alloc before, check invalid"); - GELOGE(PARAM_INVALID, "Invalid memory pointer"); + GELOGE(PARAM_INVALID, "[Check][Param] Invalid memory pointer:%p", memory_addr); return PARAM_INVALID; } it->second.second.reset(); diff --git a/ge/graph/manager/host_mem_manager.cc b/ge/graph/manager/host_mem_manager.cc index 2908df39..63265bf4 100644 --- a/ge/graph/manager/host_mem_manager.cc +++ b/ge/graph/manager/host_mem_manager.cc @@ -39,9 +39,8 @@ Status SharedMemAllocator::Allocate(SharedMemInfo &mem_info) { rtMallocHostSharedMemoryOut output_para; rtError_t rt_ret = rtMallocHostSharedMemory(&input_para, &output_para); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMallocHostSharedMemory fail, ret:0x%X", - rt_ret); - GELOGE(RT_FAILED, "Call rt api(rtMallocHostSharedMemory) failed, devid:[%u].", device_id); + REPORT_CALL_ERROR("E19999", "Call rtMallocHostSharedMemory fail, ret:0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtMallocHostSharedMemory] failed, devid:[%u].", device_id); return GE_GRAPH_MEMORY_ALLOC_FAILED; } mem_info.fd = output_para.fd; @@ -60,9 +59,8 @@ Status SharedMemAllocator::DeAllocate(SharedMemInfo &mem_info) { mem_info.host_aligned_ptr->MutableGet(), mem_info.device_address}; rtError_t rt_ret = rtFreeHostSharedMemory(&free_para); if (rt_ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtFreeHostSharedMemory fail, ret:0x%X", - rt_ret); - GELOGE(RT_FAILED, "Call rt api(rtFreeHostSharedMemory) failed, ret: 0x%X.", rt_ret); + REPORT_CALL_ERROR("E19999", "Call rtFreeHostSharedMemory fail, ret:0x%X", rt_ret); + GELOGE(RT_FAILED, "[Call][RtFreeHostSharedMemory] failed, ret:0x%X.", rt_ret); return RT_FAILED; } return ge::SUCCESS; @@ -78,7 +76,7 @@ Status HostMemManager::Initialize() { allocator_ = std::unique_ptr(new (std::nothrow) SharedMemAllocator()); if (allocator_ == nullptr) { REPORT_CALL_ERROR("E19999", "New SharedMemAllocator fail"); - GELOGE(GE_GRAPH_MALLOC_FAILED, "Shared memory allocator init failed!"); + GELOGE(GE_GRAPH_MALLOC_FAILED, "[New][SharedMemAllocator] failed!"); return GE_GRAPH_MALLOC_FAILED; } return SUCCESS; @@ -98,9 +96,8 @@ Status HostMemManager::MallocSharedMemory(SharedMemInfo &mem_info) { std::lock_guard lock(mutex_); auto iter = var_memory_base_map_.find(mem_info.op_name); if (iter != var_memory_base_map_.end()) { - REPORT_INNER_ERROR("E19999", "MemInfo.op_name:%s can't find in var_memory_base_map_", - mem_info.op_name.c_str()); - GELOGE(FAILED, "Host shared memory for op %s has been malloced", mem_info.op_name.c_str()); + REPORT_INNER_ERROR("E19999", "Host shared memory for op %s has been malloced", mem_info.op_name.c_str()); + GELOGE(FAILED, "[Check][Param] Host shared memory for op %s has been malloced", mem_info.op_name.c_str()); return FAILED; } mem_info.shm_name = OpNameToShmName(mem_info.op_name); @@ -113,9 +110,8 @@ Status HostMemManager::MallocSharedMemory(SharedMemInfo &mem_info) { Status HostMemManager::QueryVarMemInfo(const string &op_name, uint64_t &base_addr, uint64_t &data_size) { std::lock_guard lock(mutex_); if (var_memory_base_map_.find(op_name) == var_memory_base_map_.end()) { - REPORT_INNER_ERROR("E19999", "MemInfo.op_name:%s can't find in var_memory_base_map_", - op_name.c_str()); - GELOGE(INTERNAL_ERROR, "Find host base base_addr failed,node name:%s!", op_name.c_str()); + REPORT_INNER_ERROR("E19999", "MemInfo.op_name:%s can't find in var_memory_base_map_", op_name.c_str()); + GELOGE(INTERNAL_ERROR, "[Check][Param] Find host base base_addr failed, node name:%s!", op_name.c_str()); return INTERNAL_ERROR; } base_addr = static_cast(reinterpret_cast(var_memory_base_map_[op_name].device_address)); diff --git a/ge/graph/manager/memory_api.cc b/ge/graph/manager/memory_api.cc index 8e737021..8ea0594b 100644 --- a/ge/graph/manager/memory_api.cc +++ b/ge/graph/manager/memory_api.cc @@ -50,9 +50,8 @@ Status RdmaRemoteRegister(const std::vector &var_info, rtMemType_t path.append(file_name); string canonical_path = RealPath(path.c_str()); if (canonical_path.empty()) { - REPORT_INNER_ERROR("E19999", "canonical_path:%s is empty, check invalid", - canonical_path.c_str()); - GELOGE(FAILED, "Failed to get realpath of %s", path.c_str()); + REPORT_INNER_ERROR("E19999", "canonical_path:%s is empty, check invalid", canonical_path.c_str()); + GELOGE(FAILED, "[Call][RealPath] Failed to get realpath of %s", path.c_str()); return FAILED; } GELOGI("FileName:%s, Path:%s.", file_name.c_str(), canonical_path.c_str()); @@ -69,15 +68,14 @@ Status RdmaRemoteRegister(const std::vector &var_info, rtMemType_t if (hcom_remote_mem_register == nullptr) { REPORT_CALL_ERROR("E19999", "Symbol HcomRegRemoteAccessMem can't find in %s, check invalid", canonical_path.c_str()); - GELOGE(FAILED, "Failed to invoke hcom_remote_mem_register function."); + GELOGE(FAILED, "[Check][Param] Symbol HcomRegRemoteAccessMem can't find in %s", canonical_path.c_str()); return FAILED; } HcclResult hccl_ret = hcom_remote_mem_register(reg_addrs.get(), table_len); if (hccl_ret != HCCL_SUCCESS) { - REPORT_CALL_ERROR("E19999", "Call hcom_remote_mem_register failed, ret:%d,", - hccl_ret); - GELOGE(HCCL_E_INTERNAL, "Rdma mem register failed, ret: 0x%X", hccl_ret); + REPORT_CALL_ERROR("E19999", "Call hcom_remote_mem_register failed, ret:%d,", hccl_ret); + GELOGE(HCCL_E_INTERNAL, "[Call][HcomRemoteMemRegister] Rdma mem register failed, ret:0x%X", hccl_ret); return HCCL_E_INTERNAL; } return SUCCESS; @@ -88,14 +86,14 @@ Status MallocSharedMemory(const TensorInfo &tensor_info, uint64_t &dev_addr, uin uint32_t type_size = 0; bool result = TypeUtils::GetDataTypeLength(tensor_info.data_type, type_size); if (!result) { - GELOGE(GRAPH_FAILED, "GetDataTypeLength failed, data_type=(%s).", + GELOGE(GRAPH_FAILED, "[Get][DataTypeLength] failed, data_type=(%s).", TypeUtils::DataTypeToSerialString(tensor_info.data_type).c_str()); return GRAPH_FAILED; } memory_size = type_size; for (auto dim : tensor_info.dims) { if (dim <= 0) { - GELOGE(GRAPH_FAILED, "Tensor dims should be positive"); + GELOGE(GRAPH_FAILED, "[Check][Param] Tensor dims should be positive"); return GRAPH_FAILED; } memory_size *= dim; @@ -103,7 +101,7 @@ Status MallocSharedMemory(const TensorInfo &tensor_info, uint64_t &dev_addr, uin SharedMemInfo mem_info(tensor_info.var_name, memory_size); Status ret = HostMemManager::Instance().MallocSharedMemory(mem_info); if (ret != SUCCESS) { - GELOGE(GRAPH_FAILED, "MallocSharedMemory failed op name [%s]", tensor_info.var_name.c_str()); + GELOGE(GRAPH_FAILED, "[Malloc][SharedMemory] failed, op name [%s]", tensor_info.var_name.c_str()); return GRAPH_FAILED; } dev_addr = reinterpret_cast(reinterpret_cast(mem_info.device_address)); diff --git a/ge/graph/manager/model_manager/event_manager.cc b/ge/graph/manager/model_manager/event_manager.cc index 69a946f9..339e9894 100644 --- a/ge/graph/manager/model_manager/event_manager.cc +++ b/ge/graph/manager/model_manager/event_manager.cc @@ -45,7 +45,7 @@ Status EventManager::Init(size_t event_num) { void EventManager::Release() noexcept { for (size_t i = 0; i < this->event_list_.size(); ++i) { rtError_t rt_ret = rtEventDestroy(this->event_list_[i]); - RETURN_IF_COND_NOT_MET(rt_ret == RT_ERROR_NONE, "Destroy event failed, idx is %zu, ret is 0x%x.", i, rt_ret); + RETURN_IF_COND_NOT_MET(rt_ret == RT_ERROR_NONE, "[Destroy][Event] failed, idx is %zu, ret is 0x%x.", i, rt_ret); } this->event_list_.clear(); diff --git a/ge/graph/manager/rdma_pool_allocator.cc b/ge/graph/manager/rdma_pool_allocator.cc index 58829adb..4297be95 100644 --- a/ge/graph/manager/rdma_pool_allocator.cc +++ b/ge/graph/manager/rdma_pool_allocator.cc @@ -82,8 +82,8 @@ Status RdmaPoolAllocator::InitMemory(size_t mem_size) { auto device_id = GetContext().DeviceId(); GELOGD("Init Rdma Memory with size [%zu] for devid:[%u]", mem_size, device_id); if (rdma_base_addr_ != nullptr) { - REPORT_INNER_ERROR("E19999", "Param rdma_base_addr_ is nullptr, check invalid"); - GELOGE(GE_MULTI_INIT, "Rdma pool has been malloced"); + REPORT_INNER_ERROR("E19999", "Param rdma_base_addr_ is not nullptr, devid:%u, check invalid", device_id); + GELOGE(GE_MULTI_INIT, "[Check][Param] Rdma pool has been malloced, devid:%u", device_id); return GE_MULTI_INIT; } const std::string purpose = "Memory for rdma pool."; @@ -95,15 +95,15 @@ Status RdmaPoolAllocator::InitMemory(size_t mem_size) { rdma_base_addr_ = memory_allocator_->MallocMemory(purpose, mem_size, device_id); if (rdma_base_addr_ == nullptr) { - GELOGE(GE_GRAPH_MALLOC_FAILED, "Rdma pool memory malloc failed"); + GELOGE(GE_GRAPH_MALLOC_FAILED, "[Malloc][Memory] failed, size:%zu, device_id:%u", mem_size, device_id); return GE_GRAPH_MALLOC_FAILED; } rdma_mem_size_ = mem_size; // Init with a base block. auto *base_block = new (std::nothrow) Block(device_id, mem_size, rdma_base_addr_); if (base_block == nullptr) { - REPORT_CALL_ERROR("E19999", "New Block failed, device_id:%u", device_id); - GELOGE(GE_GRAPH_MALLOC_FAILED, "Block malloc failed"); + REPORT_CALL_ERROR("E19999", "New Block failed, size:%zu, device_id:%u", mem_size, device_id); + GELOGE(GE_GRAPH_MALLOC_FAILED, "[New][Block] failed, size:%zu, device_id:%u", mem_size, device_id); return GE_GRAPH_MALLOC_FAILED; } block_bin_.insert(base_block); @@ -123,7 +123,7 @@ uint8_t *RdmaPoolAllocator::Malloc(size_t size, uint32_t device_id) { if (block->ptr == nullptr) { REPORT_INNER_ERROR("E19999", "Rdmapool memory address is nullptr, device_id:%u, check invalid", device_id); - GELOGE(INTERNAL_ERROR, "Rdmapool memory address is nullptr."); + GELOGE(INTERNAL_ERROR, "[Check][Param] Rdmapool memory address is nullptr, device_id:%u", device_id); return nullptr; } allocated_blocks_.emplace(block->ptr, block); @@ -155,9 +155,8 @@ uint8_t *RdmaPoolAllocator::Malloc(size_t size, uint32_t device_id) { Status RdmaPoolAllocator::Free(uint8_t *memory_addr, uint32_t device_id) { GELOGI("Free rdma memory, device id = %u", device_id); if (memory_addr == nullptr) { - REPORT_INNER_ERROR("E19999", "Param memory_addr is nullptr, device_id:%u, check invalid", - device_id); - GELOGE(GE_GRAPH_FREE_FAILED, "Invalid memory pointer"); + REPORT_INNER_ERROR("E19999", "Param memory_addr is nullptr, device_id:%u, check invalid", device_id); + GELOGE(GE_GRAPH_FREE_FAILED, "[Check][Param] Invalid memory pointer, device id:%u", device_id); return GE_GRAPH_FREE_FAILED; } @@ -166,7 +165,7 @@ Status RdmaPoolAllocator::Free(uint8_t *memory_addr, uint32_t device_id) { if (it == allocated_blocks_.end()) { REPORT_INNER_ERROR("E19999", "Param memory_addr is not allocated before, device_id:%u, " "check invalid", device_id); - GELOGE(PARAM_INVALID, "Invalid memory pointer"); + GELOGE(PARAM_INVALID, "[Check][Param] Invalid memory pointer, device id:%u", device_id); return PARAM_INVALID; } @@ -209,7 +208,7 @@ void RdmaPoolAllocator::MergeBlocks(Block *dst, Block *src) { Status RdmaPoolAllocator::GetBaseAddr(uint64_t &base_addr, uint64_t &mem_size) { if (rdma_base_addr_ == nullptr) { REPORT_INNER_ERROR("E19999", "Param rdma_base_addr_ is nullptr, check invalid"); - GELOGE(INTERNAL_ERROR, "Rdma base addr is nullptr."); + GELOGE(INTERNAL_ERROR, "[Check][Param] Rdma base addr is nullptr."); return INTERNAL_ERROR; } base_addr = static_cast(reinterpret_cast(rdma_base_addr_)); diff --git a/ge/graph/manager/trans_var_data_utils.cc b/ge/graph/manager/trans_var_data_utils.cc index 9c1290fa..621eba79 100644 --- a/ge/graph/manager/trans_var_data_utils.cc +++ b/ge/graph/manager/trans_var_data_utils.cc @@ -37,7 +37,8 @@ class RtContextSwitchGuard { if (ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtCtxGetCurrent failed, device_id:%u, ret:0x%X,", device_id, ret); - GELOGE(RT_FAILED, "Failed to get current context from rt, error-code %d", ret); + GELOGE(RT_FAILED, "[Call][RtCtxGetCurrent] Failed to get current context, device_id:%u, ret:0x%X", + device_id, ret); return; } @@ -45,15 +46,14 @@ class RtContextSwitchGuard { if (ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtCtxCreate failed, device_id:%u, ret:0x%X,", device_id, ret); - GELOGE(RT_FAILED, "Failed to create new context for device %u, error-code %d", device_id, ret); + GELOGE(RT_FAILED, "[Call][RtCtxCreate] Failed to create new context for device:%u, ret:%d", device_id, ret); return; } ret = rtCtxSetCurrent(current_); if (ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtCtxSetCurrent failed, device_id:%u, ret:0x%X,", - device_id, ret); - GELOGE(RT_FAILED, "Failed to switch context to normal, context %p, device %u", current_, device_id); + REPORT_CALL_ERROR("E19999", "Call rtCtxSetCurrent failed, device_id:%u, ret:0x%X", device_id, ret); + GELOGE(RT_FAILED, "[Call][RtCtxSetCurrent] failed, device_id:%u, ret:0x%X", device_id, ret); return; } GELOGD("Create and switch rt context %p type %d for device %u, backup last %p.", current_, mode, device_id, last_); @@ -80,7 +80,7 @@ int64_t CalcVarSizeInBytes(const GeTensorDesc &desc) { if (var_size <= 0) { REPORT_INNER_ERROR("E19999", "Data type:%s in desc, it's size:%ld < 0, check invalid", TypeUtils::DataTypeToSerialString(desc.GetDataType()).c_str(), var_size); - GELOGE(PARAM_INVALID, "Failed to calc var data size from data type %s", + GELOGE(PARAM_INVALID, "[Calc][VarDataSize] by data type %s failed.", TypeUtils::DataTypeToSerialString(desc.GetDataType()).c_str()); return -1; } @@ -99,7 +99,8 @@ Status CopyVarToDevice(const NodePtr &var, const formats::TransResult &trans_res if (ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, op:%s(%s), size:%lu, ret:0x%X,", var->GetName().c_str(), var->GetType().c_str(), trans_result.length, ret); - GELOGE(RT_FAILED, "Failed to copy memory to device, size %zu", trans_result.length); + GELOGE(RT_FAILED, "[Call][RtMemcpy] failed, op:%s(%s), size:%lu, ret:0x%X,", var->GetName().c_str(), + var->GetType().c_str(), trans_result.length, ret); return RT_FAILED; } return SUCCESS; @@ -111,21 +112,17 @@ Status CopyVarFromDevice(uint64_t session_id, const NodePtr &var, std::unique_pt GE_CHECK_NOTNULL(var); auto ret = VarManager::Instance(session_id)->GetVarAddr(var->GetName(), input_desc, &var_logic); if (ret != SUCCESS) { - GELOGE(INTERNAL_ERROR, - "Failed to copy var %s from device, can not find it" - " from var manager %u", - var->GetName().c_str(), ret); + GELOGE(INTERNAL_ERROR, "[Get][VarAddr] failed, node:%s, session_id:%lu, ret:%d", + var->GetName().c_str(), session_id, ret); return INTERNAL_ERROR; } uint8_t *var_addr = VarManager::Instance(session_id)->GetVarMemoryAddr(var_logic, RT_MEMORY_HBM); if (var_addr == nullptr) { - REPORT_CALL_ERROR("E19999", "Get variable memory addr failed, mem_type:%d, op:%s(%s), session_id:%lu,", + REPORT_CALL_ERROR("E19999", "Get variable memory addr failed, mem_type:%d, op:%s(%s), session_id:%lu", RT_MEMORY_HBM, var->GetName().c_str(), var->GetType().c_str(), session_id); - GELOGE(INTERNAL_ERROR, - "Failed to copy var %s from device, cant not get " - "var addr from logic addr %p", - var->GetName().c_str(), var_logic); + GELOGE(INTERNAL_ERROR, "[Get][VarMemoryAddr] failed, mem_type:%d, op:%s(%s), session_id:%lu", + RT_MEMORY_HBM, var->GetName().c_str(), var->GetType().c_str(), session_id); return INTERNAL_ERROR; } @@ -136,9 +133,10 @@ Status CopyVarFromDevice(uint64_t session_id, const NodePtr &var, std::unique_pt std::unique_ptr var_host(new(std::nothrow) uint8_t[var_size_bytes]); if (var_host == nullptr) { - REPORT_CALL_ERROR("E19999", "New host memory failed, size:%ld, op:%s(%s), session_id:%lu,", + REPORT_CALL_ERROR("E19999", "New host memory failed, size:%ld, op:%s(%s), session_id:%lu", var_size_bytes, var->GetName().c_str(), var->GetType().c_str(), session_id); - GELOGE(OUT_OF_MEMORY, "Failed to malloc rt-host memory, size %ld", var_size_bytes); + GELOGE(OUT_OF_MEMORY, "[New][Memory] for rt-host failed, size:%ld, op:%s(%s), session_id:%lu", + var_size_bytes, var->GetName().c_str(), var->GetType().c_str(), session_id); return OUT_OF_MEMORY; } @@ -147,10 +145,8 @@ Status CopyVarFromDevice(uint64_t session_id, const NodePtr &var, std::unique_pt if (ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%ld, op:%s(%s), session_id:%lu, ret:0x%X", var_size_bytes, var->GetName().c_str(), var->GetType().c_str(), session_id, ret); - GELOGE(RT_FAILED, - "Failed to copy var memory from device, var %s, size %ld," - " rt-error-code %u", - var->GetName().c_str(), var_size_bytes, ret); + GELOGE(RT_FAILED, "[Call][RtMemcpy] failed, size:%ld, op:%s(%s), session_id:%lu, ret:0x%X", + var_size_bytes, var->GetName().c_str(), var->GetType().c_str(), session_id, ret); return RT_FAILED; } @@ -197,9 +193,7 @@ Status TransVarOnHost(uint8_t *var_data, const VarTransRoad &trans_road, formats formats::ShapeToString(src_shape).c_str(), formats::ShapeToString(dst_shape).c_str(), TypeUtils::DataTypeToSerialString(data_type).c_str(), ret); - GELOGE(INTERNAL_ERROR, - "Failed to trans format from %s to %s, shape %s to %s, " - "data type %s error code %u", + GELOGE(INTERNAL_ERROR, "[Trans][Format] from %s to %s, shape %s to %s failed, data type %s error code %u", TypeUtils::FormatToSerialString(src_format).c_str(), TypeUtils::FormatToSerialString(dst_format).c_str(), formats::ShapeToString(src_shape).c_str(), formats::ShapeToString(dst_shape).c_str(), TypeUtils::DataTypeToSerialString(data_type).c_str(), ret); @@ -221,7 +215,7 @@ Status TransVarOnHost(uint8_t *var_data, const VarTransRoad &trans_road, formats TypeUtils::DataTypeToSerialString(src_data_type).c_str(), TypeUtils::DataTypeToSerialString(dst_data_type).c_str(), formats::ShapeToString(input_shape).c_str(), src_data_size, ret); - GELOGE(INTERNAL_ERROR, "Failed to trans data type from %s to %s, input shape %s, data size %ld, error code %u", + GELOGE(INTERNAL_ERROR, "[Trans][DataType] from %s to %s failed, input shape %s, data size %ld, error code %u", TypeUtils::DataTypeToSerialString(src_data_type).c_str(), TypeUtils::DataTypeToSerialString(dst_data_type).c_str(), formats::ShapeToString(input_shape).c_str(), src_data_size, ret); @@ -230,7 +224,7 @@ Status TransVarOnHost(uint8_t *var_data, const VarTransRoad &trans_road, formats } else { REPORT_INNER_ERROR("E19999", "Trans var data failed, the trans type %s does not supported, check invalid", trans_info.node_type.c_str()); - GELOGE(UNSUPPORTED, "Failed to trans var data, the trans type %s does not supported", + GELOGE(UNSUPPORTED, "[Trans][VarData] failed, the trans type %s does not supported", trans_info.node_type.c_str()); return UNSUPPORTED; } @@ -255,10 +249,8 @@ Status ReAssignVarAddr(uint64_t session_id, uint8_t *var_logic = nullptr; Status ret = VarManager::Instance(session_id)->GetVarAddr(var_name, tensor_desc, &var_logic); if (ret != SUCCESS) { - GELOGE(INTERNAL_ERROR, - "Failed to get var %s device addr, can not find it" - " from var manager %u", - var_name.c_str(), ret); + GELOGE(INTERNAL_ERROR, "[Get][VarAddr] failed, var name:%s, session_id:%lu, ret:%u", + var_name.c_str(), session_id, ret); return INTERNAL_ERROR; } @@ -266,7 +258,8 @@ Status ReAssignVarAddr(uint64_t session_id, if (var_addr == nullptr) { REPORT_CALL_ERROR("E19999", "Get variable memory addr failed, mem_type:%d, var_name:%s, session_id:%lu,", RT_MEMORY_HBM, var_name.c_str(), session_id); - GELOGE(INTERNAL_ERROR, "Failed to convert var %s logic addr to real addr", var_name.c_str()); + GELOGE(INTERNAL_ERROR, "[Get][VarMemoryAddr] failed, mem_type:%d, var_name:%s, session_id:%lu", + RT_MEMORY_HBM, var_name.c_str(), session_id); return INTERNAL_ERROR; } *var_device = var_addr; @@ -293,9 +286,8 @@ Status TransVarData(const NodePtr &var, const VarTransRoad &trans_road, uint64_t // Sync var data from device std::unique_ptr var_data; if (trans_road.empty()) { - REPORT_INNER_ERROR("E19999", "Param trans_road is empty, session_id:%lu, check invalid", - session_id); - GELOGE(INTERNAL_ERROR, "Failed to get trans_road, trans_road is empty."); + REPORT_INNER_ERROR("E19999", "Param trans_road is empty, session_id:%lu, check invalid", session_id); + GELOGE(INTERNAL_ERROR, "[Check][Param] trans_road is empty, session_id:%lu", session_id); return INTERNAL_ERROR; } const GeTensorDesc &input_desc = trans_road.begin()->input; @@ -307,7 +299,7 @@ Status TransVarData(const NodePtr &var, const VarTransRoad &trans_road, uint64_t formats::TransResult trans_result{}; ret = TransVarOnHost(var_data.get(), trans_road, trans_result); if (ret != SUCCESS) { - GELOGE(ret, "Failed to trans var data on host, error code %u", ret); + GELOGE(ret, "[Call][TransVarOnHost] failed, session_id:%lu, ret:%u", session_id, ret); return ret; } @@ -319,14 +311,15 @@ Status TransVarData(const NodePtr &var, const VarTransRoad &trans_road, uint64_t /// TensorDesc needs to be removed. This change is large and needs to be performed step by step. ret = ReAssignVarAddr(session_id, var->GetName(), trans_road.rbegin()->output, &var_device); if (ret != SUCCESS) { - GELOGE(ret, "Failed to re-assign memory on device, size %zu", trans_result.length); + GELOGE(ret, "[Call][ReAssignVarAddr] failed, session id:%lu, op:%s, ret:%u", + session_id, var->GetName().c_str(), ret); return ret; } // sync new data to device ret = CopyVarToDevice(var, trans_result, var_device); if (ret != SUCCESS) { - GELOGE(ret, "Failed to send var data to device"); + GELOGE(ret, "[Call][CopyVarToDevice] failed, var:%s, ret:%u", var->GetName().c_str(), ret); return ret; } @@ -350,7 +343,10 @@ Status TransTensor(uint8_t *var_data, const NodePtr &var_src, const NodePtr &var TypeUtils::DataTypeToSerialString(src_data_datatype).c_str(), TypeUtils::DataTypeToSerialString(dst_data_datatype).c_str(), src_data_shape_size, ret); - GELOGE(INTERNAL_ERROR, "trans var data on host failed"); + GELOGE(INTERNAL_ERROR, "[Trans][DataType] from %s to %s failed, data size %ld, ret:%u", + TypeUtils::DataTypeToSerialString(src_data_datatype).c_str(), + TypeUtils::DataTypeToSerialString(dst_data_datatype).c_str(), + src_data_shape_size, ret); return ret; }); return SUCCESS; @@ -366,9 +362,11 @@ Status CopyTensorFromSrcVarNode(const NodePtr &var_src, /// need copy value from var_fp32 to var_fp16. /// [opdesc of var_src and var_dst are checked before passed in, no need to check if they are nullptr] GE_IF_BOOL_EXEC(var_src == nullptr || var_dst == nullptr, - REPORT_INNER_ERROR("E19999", "Param var_src or var_dst is empty, session_id:%lu, device_id:%u, " + REPORT_INNER_ERROR("E19999", "Param var_src or var_dst is nullptr, session_id:%lu, device_id:%u, " "check invalid", session_id, device_id); - GELOGE(FAILED, "node var is nullptr"); return FAILED); + GELOGE(FAILED, "[Check][Param] Param var_src or var_dst is nullptr, session_id:%lu, device_id:%u", + session_id, device_id); + return FAILED); // src_node output_desc (fp32) GeTensorDesc output_desc = var_src->GetOpDesc()->GetOutputDesc(0); auto src_data_type = output_desc.GetDataType(); @@ -390,31 +388,45 @@ Status CopyTensorFromSrcVarNode(const NodePtr &var_src, RtContextSwitchGuard switch_context(RT_CTX_NORMAL_MODE, device_id); // copy from src_node auto ret = CopyVarFromDevice(session_id, var_src, var_src_data, output_desc); - GE_IF_BOOL_EXEC(ret != SUCCESS, GELOGE(FAILED, "Copy Var From Device failed"); return ret); + GE_IF_BOOL_EXEC(ret != SUCCESS, + GELOGE(FAILED, "[Call][CopyVarFromDevice] failed, session id:%lu, var_src:%s", + session_id, var_src->GetName().c_str()); + return ret); // trans dtype formats::TransResult trans_result{}; ret = TransTensor(var_src_data.get(), var_src, var_dst, trans_result); - GE_IF_BOOL_EXEC(ret != SUCCESS, GELOGE(INTERNAL_ERROR, "trans var data on host failed"); return ret); + GE_IF_BOOL_EXEC(ret != SUCCESS, + GELOGE(INTERNAL_ERROR, "[Trans][Tensor] failed, var_src:%s, var_dst:%s", + var_src->GetName().c_str(), var_dst->GetName().c_str()); + return ret); // reset src value. void *var_device = nullptr; ret = ReAssignVarAddr(session_id, var_dst->GetName(), dst_tensor_desc, &var_device); - GE_IF_BOOL_EXEC(ret != SUCCESS, GELOGE(INTERNAL_ERROR, "assign mem failed"); return ret); + GE_IF_BOOL_EXEC(ret != SUCCESS, + GELOGE(INTERNAL_ERROR, "[Call][ReAssignVarAddr] failed, session id:%lu, var_dst:%s", + session_id, var_dst->GetName().c_str()); + return ret); // copy to device ret = CopyVarToDevice(var_dst, trans_result, var_device); - GE_IF_BOOL_EXEC(ret != SUCCESS, GELOGE(ret, "Failed to send var data to device"); return ret); + GE_IF_BOOL_EXEC(ret != SUCCESS, + GELOGE(ret, "[Call][CopyVarToDevice] failed, var_dst:%s, ret:%u", + var_dst->GetName().c_str(), ret); + return ret); return SUCCESS; } } // namespace Status TransVarDataUtils::SyncVarData2BroadCast(const string &var_name, const ge::GeTensorDesc &src_tensor_desc, uint8_t *dst_addr, int64_t dst_addr_size, uint64_t session_id) { - GE_CHK_BOOL_RET_STATUS(dst_addr != nullptr, FAILED, "dst addr is null. "); + GE_CHK_BOOL_RET_STATUS(dst_addr != nullptr, FAILED, "[Check][Param] dst addr is nullptr."); uint8_t *src_host_addr = nullptr; int64_t src_addr_size = 0; GE_MAKE_GUARD_RTMEM(src_host_addr); GE_CHK_STATUS_RET(SyncTensorToHost(var_name, src_tensor_desc, &src_host_addr, src_addr_size, session_id)); GELOGI("src_addr_size: %ld, dst_addr_size: %ld", src_addr_size, dst_addr_size); - GE_CHK_BOOL_RET_STATUS(src_addr_size == dst_addr_size, FAILED, "var data size is not equal broadcast "); + GE_CHK_BOOL_RET_STATUS(src_addr_size == dst_addr_size, FAILED, + "[Check][Param] src_addr_size:%ld not equal to dst_addr_size:%ld", + src_addr_size, dst_addr_size); GE_CHK_RT_RET(rtMemcpy(dst_addr, dst_addr_size, src_host_addr, src_addr_size, RT_MEMCPY_HOST_TO_DEVICE)); return SUCCESS; @@ -422,7 +434,7 @@ Status TransVarDataUtils::SyncVarData2BroadCast(const string &var_name, const ge Status TransVarDataUtils::SyncBroadCastData2Var(uint8_t *src_addr, int64_t src_addr_size, const string &var_name, const ge::GeTensorDesc &dst_tensor_desc, uint64_t session_id) { - GE_CHK_BOOL_RET_STATUS(src_addr != nullptr, FAILED, "src addr is null. "); + GE_CHK_BOOL_RET_STATUS(src_addr != nullptr, FAILED, "[Check][Param] src addr is nullptr. "); uint8_t *host_addr = nullptr; GE_MAKE_GUARD_RTMEM(host_addr); GE_CHK_RT_RET(rtMallocHost(reinterpret_cast(&host_addr), src_addr_size)); @@ -436,7 +448,7 @@ Status TransVarDataUtils::SyncBroadCastData2Var(uint8_t *src_addr, int64_t src_a Status TransVarDataUtils::SyncTensorToHost(const string &var_name, const ge::GeTensorDesc &src_tensor_desc, uint8_t **host_addr, int64_t &src_tensor_size, uint64_t session_id) { - GE_CHK_STATUS_RET(ge::TensorUtils::GetSize(src_tensor_desc, src_tensor_size), "get size from TensorDesc failed"); + GE_CHK_STATUS_RET(ge::TensorUtils::GetSize(src_tensor_desc, src_tensor_size), "[Get][Size] from TensorDesc failed"); uint8_t *src_addr = nullptr; GE_CHK_STATUS_RET(VarManager::Instance(session_id)->GetVarAddr(var_name, src_tensor_desc, &src_addr)); @@ -493,7 +505,8 @@ Status TransVarDataUtils::TransAllVarData(const vector &variable_nodes, if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtCtxSetCurrent failed, session_id:%lu, graph_id:%u, ret:0x%X,", session_id, graph_id, rt_ret); - GELOGE(RT_FAILED, "Failed to set context, error_code is: 0x%X.", rt_ret); + GELOGE(RT_FAILED, "[Call][RtCtxSetCurrent] failed, session_id:%lu, graph_id:%u, ret:0x%X,", + session_id, graph_id, rt_ret); return RT_ERROR_TO_GE_STATUS(rt_ret); } uint32_t allocated_graph_id = 0; @@ -501,8 +514,8 @@ Status TransVarDataUtils::TransAllVarData(const vector &variable_nodes, if (ret != SUCCESS) { REPORT_CALL_ERROR("E19999", "Get allocated GraphId failed, session_id:%lu, graph_id:%u, ret:0x%X,", session_id, graph_id, ret); - GELOGE(INTERNAL_ERROR, "var has not been allocated, node:%s, graph_id:%u.", node->GetName().c_str(), - graph_id); + GELOGE(INTERNAL_ERROR, "[Get][AllocatedGraphId] failed, node:%s, graph_id:%u.", + node->GetName().c_str(), graph_id); return INTERNAL_ERROR; } uint32_t changed_graph_id = 0; @@ -518,7 +531,8 @@ Status TransVarDataUtils::TransAllVarData(const vector &variable_nodes, } ret = TransVarData(node, *trans_road, session_id); if (ret != SUCCESS) { - GELOGE(INTERNAL_ERROR, "TransVarData failed, node:%s, graph_id:%u.", node->GetName().c_str(), graph_id); + GELOGE(INTERNAL_ERROR, "[Trans][VarData] failed, node:%s, graph_id:%u, session_id:%lu.", + node->GetName().c_str(), graph_id, session_id); return INTERNAL_ERROR; } VarManager::Instance(session_id)->RemoveChangedGraphId(node->GetName()); @@ -527,7 +541,7 @@ Status TransVarDataUtils::TransAllVarData(const vector &variable_nodes, }, node, session_id, context, graph_id, ErrorManager::GetInstance().GetErrorManagerContext()); if (!f.valid()) { - GELOGE(FAILED, "Future is invalid"); + GELOGE(FAILED, "[Check][Param] Future is invalid, session id:%lu, graph id:%u", session_id, graph_id); return FAILED; } vector_future.push_back(std::move(f)); @@ -537,7 +551,7 @@ Status TransVarDataUtils::TransAllVarData(const vector &variable_nodes, for (size_t i = 0; i < vector_future.size(); ++i) { ret_status = vector_future[i].get(); if (ret_status != SUCCESS) { - GELOGE(ret_status, "TransAllVarData:: trans %zu vardata failed", i); + GELOGE(ret_status, "[Check][Param] trans %zu vardata failed", i); return ret_status; } } @@ -550,7 +564,8 @@ Status TransVarDataUtils::CopyVarData(const ComputeGraphPtr &compute_graph, uint if (compute_graph == nullptr) { REPORT_INNER_ERROR("E19999", "Param compute_graph is nullptr, session_id:%lu, device_id:%u, check invalid", session_id, device_id); - GELOGE(FAILED, "compute_graph is nullptr"); + GELOGE(FAILED, "[Check][Param] compute_graph is nullptr, session_id:%lu, device_id:%u", + session_id, device_id); return FAILED; } @@ -568,7 +583,10 @@ Status TransVarDataUtils::CopyVarData(const ComputeGraphPtr &compute_graph, uint GELOGI("current_var_node__: [%s] copy_from_var_node__: [%s].", node->GetName().c_str(), src_node->GetName().c_str()); auto ret = CopyTensorFromSrcVarNode(src_node, node, session_id, device_id); - GE_IF_BOOL_EXEC(ret != SUCCESS, GELOGE(FAILED, "copy tensor failed!"); return FAILED); + GE_IF_BOOL_EXEC(ret != SUCCESS, + GELOGE(FAILED, "[Copy][Tensor] failed, src_node:%s, node:%s, session_id:%lu, device_id:%u", + src_node->GetName().c_str(), node->GetName().c_str(), session_id, device_id); + return FAILED); // only copy once (void) ge::AttrUtils::SetBool(node->GetOpDesc(), "_copy_value", true); // no need to check value } diff --git a/ge/graph/manager/util/debug.cc b/ge/graph/manager/util/debug.cc index 1dd97bc1..d20280fb 100644 --- a/ge/graph/manager/util/debug.cc +++ b/ge/graph/manager/util/debug.cc @@ -63,17 +63,15 @@ Status Debug::DumpDevMem(const char *file, const void *addr, int64_t size) { uint8_t *host_addr = nullptr; rtError_t ret = rtMallocHost(reinterpret_cast(&host_addr), size); if (ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMallocHost failed, size:%zu, ret: 0x%X", - size, ret); - GELOGE(FAILED, "Call rt api rtMallocHost failed, ret: 0x%X", ret); + REPORT_CALL_ERROR("E19999", "Call rtMallocHost failed, size:%zu, ret:0x%X", size, ret); + GELOGE(FAILED, "[Call][RtMallocHost] failed, size:%zu, ret:0x%X", size, ret); return FAILED; } GE_MAKE_GUARD_RTMEM(host_addr); ret = rtMemcpy(host_addr, size, addr, size, RT_MEMCPY_DEVICE_TO_HOST); if (ret != RT_ERROR_NONE) { - REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%zu, ret: 0x%X", - size, ret); - GELOGE(FAILED, "Call rt api rtMemcpy failed, ret: 0x%X", ret); + REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%zu, ret:0x%X", size, ret); + GELOGE(FAILED, "[Call][RtMemcpy] failed, size:%zu, ret:0x%X", size, ret); return FAILED; } diff --git a/ge/graph/manager/util/hcom_util.cc b/ge/graph/manager/util/hcom_util.cc index a30321f9..2da19cc9 100644 --- a/ge/graph/manager/util/hcom_util.cc +++ b/ge/graph/manager/util/hcom_util.cc @@ -28,7 +28,8 @@ Status HcomOmeUtil::GetHcclDataType(const ge::ConstOpDescPtr &op_desc, std::vector &kernel_hccl_infos) { GE_CHECK_NOTNULL(op_desc); if (CheckKernelHcclInfo(op_desc, kernel_hccl_infos) != SUCCESS) { - GELOGE(PARAM_INVALID, "HcomOmeUtil:: the number of GETaskKernelHcclInfo is invalid."); + GELOGE(PARAM_INVALID, "[Check][KernelHcclInfo] failed, op:%s(%s).", + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return PARAM_INVALID; } GELOGI("GetHcclDataType start, node[%s], opType[%s].", op_desc->GetName().c_str(), op_desc->GetType().c_str()); @@ -40,10 +41,10 @@ Status HcomOmeUtil::GetHcclDataType(const ge::ConstOpDescPtr &op_desc, if (op_desc->GetType() == HCOMRECEIVE) { bool ret = ge::AttrUtils::GetDataType(op_desc, HCOM_ATTR_DATA_TYPE, src_data_type); if (ret == false) { - REPORT_INNER_ERROR("E19999", "Get Attr:%s in op:%s(%s) fail", - HCOM_ATTR_DATA_TYPE.c_str(), + REPORT_INNER_ERROR("E19999", "Get Attr:%s in op:%s(%s) fail", HCOM_ATTR_DATA_TYPE.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(PARAM_INVALID, "op:HcomReceive, op desc no attr: dtype."); + GELOGE(PARAM_INVALID, "[Get][Attr] %s in op:%s(%s) fail", HCOM_ATTR_DATA_TYPE.c_str(), + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return PARAM_INVALID; } } else { @@ -55,13 +56,11 @@ Status HcomOmeUtil::GetHcclDataType(const ge::ConstOpDescPtr &op_desc, auto iter = kConstOpHcclDataType.find(static_cast(src_data_type)); if (iter == kConstOpHcclDataType.end()) { REPORT_INNER_ERROR("E19999", "Attr:%s in op:%s(%s), value data_type:%s, not support in kConstOpHcclDataType now, " - "check invalid", HCOM_ATTR_DATA_TYPE.c_str(), - op_desc->GetName().c_str(), op_desc->GetType().c_str(), - ge::TypeUtils::DataTypeToSerialString(src_data_type).c_str()); - GELOGE(PARAM_INVALID, - "HcomOmeUtil:: Node: %s Optype: %s HcomDataType cann't support! Current Davinci Data Type : %s", - op_desc->GetName().c_str(), op_desc->GetType().c_str(), - ge::TypeUtils::DataTypeToSerialString(src_data_type).c_str()); + "check invalid", HCOM_ATTR_DATA_TYPE.c_str(), op_desc->GetName().c_str(), + op_desc->GetType().c_str(), ge::TypeUtils::DataTypeToSerialString(src_data_type).c_str()); + GELOGE(PARAM_INVALID, "[Check][Param] Attr:%s in op:%s(%s), value data_type:%s, " + "not support in kConstOpHcclDataType now", HCOM_ATTR_DATA_TYPE.c_str(), op_desc->GetName().c_str(), + op_desc->GetType().c_str(), ge::TypeUtils::DataTypeToSerialString(src_data_type).c_str()); return PARAM_INVALID; } @@ -73,7 +72,7 @@ Status HcomOmeUtil::GetHcclDataType(const ge::ConstOpDescPtr &op_desc, Status HcomOmeUtil::GetHcclTypeSize(HcclDataType data_type, int32_t &size) { auto iter = kConstOpHcclDataTypeSize.find(data_type); GE_CHK_BOOL_EXEC(iter != kConstOpHcclDataTypeSize.end(), return PARAM_INVALID, - "HcomOmeUtil::HcomDataTypeSize , No DataTypeSize!"); + "[Check][Param] param data_type:%d not find", data_type); size = iter->second; return SUCCESS; @@ -83,21 +82,22 @@ Status HcomOmeUtil::GetHcomCount(const ge::ConstOpDescPtr &op_desc, HcclDataType int &count) { GE_CHECK_NOTNULL(op_desc); if (!IsHCOMOp(op_desc->GetType())) { - REPORT_INNER_ERROR("E19999", "Op:%s(%s) is not hcom op, check invalid", - op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(PARAM_INVALID, "HcomOmeUtil:: operator is not Hcom operator."); + REPORT_INNER_ERROR("E19999", "Op:%s(%s) is not hcom op, check invalid", op_desc->GetName().c_str(), + op_desc->GetType().c_str()); + GELOGE(PARAM_INVALID, "[Check][Param] Op:%s(%s) is not hcom op", op_desc->GetName().c_str(), + op_desc->GetType().c_str()); return PARAM_INVALID; } int64_t total_size = 0; int64_t align_size = 512; int32_t size = 0; - GE_CHK_STATUS_RET(HcomOmeUtil::GetHcclTypeSize(data_type, size), "GetHcomCount: GetHcclTypeSize fail!"); + GE_CHK_STATUS_RET(HcomOmeUtil::GetHcclTypeSize(data_type, size), "[Get][HcclTypeSize] fail, datatype:%d", data_type); if (op_desc->GetType() == HCOMRECEIVE) { for (size_t i = 0; i < op_desc->GetOutputsSize(); ++i) { int64_t output_size = 0; GE_CHECK_NOTNULL(op_desc->GetOutputDescPtr(i)); GE_CHK_STATUS_RET(ge::TensorUtils::GetSize(*op_desc->GetOutputDescPtr(i), output_size), - "Get size from TensorDesc failed, op: %s, output index: %zu.", op_desc->GetName().c_str(), i); + "[Get][Size] from TensorDesc failed, op:%s, output index:%zu.", op_desc->GetName().c_str(), i); output_size = (output_size + align_size - 1) / align_size * align_size; total_size += output_size; } @@ -107,42 +107,48 @@ Status HcomOmeUtil::GetHcomCount(const ge::ConstOpDescPtr &op_desc, HcclDataType int64_t block_size = 0; GE_CHECK_NOTNULL(op_desc->GetInputDescPtr(i)); GE_CHK_STATUS_RET(ge::TensorUtils::GetSize(*op_desc->GetInputDescPtr(i), input_size), - "get size from TensorDesc failed, op : %s, input index : %zu", op_desc->GetName().c_str(), i); + "[Get][Size] from TensorDesc failed, op:%s, input index:%zu", op_desc->GetName().c_str(), i); // dynamic shape hccl op get size from output tensor desc if (op_desc->HasAttr(ATTR_NAME_IS_UNKNOWN_SHAPE)) { GE_CHECK_NOTNULL(op_desc->GetOutputDescPtr(i)); GE_CHK_STATUS_RET(ge::TensorUtils::GetSize(*op_desc->GetOutputDescPtr(i), input_size), - "get size from TensorDesc failed, op : %s, input index : %zu", op_desc->GetName().c_str(), i); + "[Get][Size] from TensorDesc failed, op:%s, input index:%zu", op_desc->GetName().c_str(), i); } GE_IF_BOOL_EXEC( op_desc->GetType() == HCOMREDUCESCATTER, int32_t rank_size = 0; GE_CHK_BOOL_RET_STATUS(ge::AttrUtils::GetInt(op_desc, HCOM_ATTR_RANK_SIZE, rank_size), PARAM_INVALID, - "get HCOM_ATTR_RANK_SIZE failed"); - GE_CHK_BOOL_RET_STATUS(rank_size != 0, PARAM_INVALID, "rank size is zero"); - int64_t shape_size = op_desc->GetInputDescPtr(i)->GetShape().GetShapeSize(); GE_CHK_STATUS_RET( - ge::CheckInt64Uint32MulOverflow(shape_size, size), "Product of shape size and size beyond INT64_MAX"); + "[Get][Attr] %s in op:%s(%s) failed", HCOM_ATTR_RANK_SIZE.c_str(), + op_desc->GetName().c_str(), op_desc->GetType().c_str()); + GE_CHK_BOOL_RET_STATUS(rank_size != 0, PARAM_INVALID, "[Check][Param] rank size is zero"); + int64_t shape_size = op_desc->GetInputDescPtr(i)->GetShape().GetShapeSize(); + GE_CHK_STATUS_RET(ge::CheckInt64Uint32MulOverflow(shape_size, size), + "[Check][Param] Product of shape size:%ld and size:%d beyond INT64_MAX, op:%s(%s)", + shape_size, size, op_desc->GetName().c_str(), op_desc->GetType().c_str()); block_size = (shape_size * size) / rank_size; - GE_CHK_STATUS_RET(ge::CheckInt64AddOverflow(total_size, block_size), "Total size is beyond the INT64_MAX"); + GE_CHK_STATUS_RET(ge::CheckInt64AddOverflow(total_size, block_size), + "[Check][Param] Total size:%ld is beyond the INT64_MAX, op:%s(%s)", + total_size, op_desc->GetName().c_str(), op_desc->GetType().c_str()); total_size = total_size + block_size; continue;); int64_t shape_size = op_desc->GetInputDescPtr(i)->GetShape().GetShapeSize(); GELOGD("hcom util node %s inputsize %ld, shapesize %ld, datasize %d.", op_desc->GetName().c_str(), input_size, shape_size, size); GE_CHK_STATUS_RET(ge::CheckInt64Int32MulOverflow(shape_size, size), - "Product of shape size and size beyond INT64_MAX"); + "[Check][Param] Product of shape size:%ld and size:%d beyond INT64_MAX", shape_size, size); GE_IF_BOOL_EXEC(is_allgather, block_size = shape_size * size;); GE_IF_BOOL_EXEC(!is_allgather, block_size = (input_size + align_size - 1) / align_size * align_size;); - GE_CHK_STATUS_RET(ge::CheckInt64AddOverflow(total_size, block_size), "Total size is beyond the INT64_MAX"); + GE_CHK_STATUS_RET(ge::CheckInt64AddOverflow(total_size, block_size), + "[Check][Param] Total size:%ld is beyond the INT64_MAX", total_size); total_size = total_size + block_size; } } - GE_CHK_BOOL_RET_STATUS(size != 0, PARAM_INVALID, "Size is zero"); + GE_CHK_BOOL_RET_STATUS(size != 0, PARAM_INVALID, "[Check][Param] Size is zero"); count = static_cast(total_size / size); - GE_CHK_BOOL_EXEC(total_size % size == 0, return PARAM_INVALID, "total_size:%ld is not divisiable by size:%d.", - total_size, size); + GE_CHK_BOOL_EXEC(total_size % size == 0, return PARAM_INVALID, + "[Check][Param] total_size:%ld is not divisiable by size:%d.", total_size, size); return SUCCESS; } @@ -153,32 +159,34 @@ Status HcomOmeUtil::GetHorovodCount(const ge::ConstOpDescPtr &op_desc, if (!IsHorovodOp(op_desc->GetType())) { REPORT_INNER_ERROR("E19999", "Op:%s(%s) is not horovod op, check invalid", op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(PARAM_INVALID, "HcomOmeUtil:: operator is not Horovod operator."); + GELOGE(PARAM_INVALID, "[Call][IsHorovodOp] failed, Op:%s(%s) is not horovod op", + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return PARAM_INVALID; } int64_t align_size = 512; int32_t size = 0; for (size_t i = 0; i < op_desc->GetInputsSize(); i++) { GE_CHK_STATUS_RET(HcomOmeUtil::GetHcclTypeSize(static_cast(kernel_hccl_infos[i].dataType), size), - "GetHorovodCount: GetHcclTypeSize fail!"); + "[Call][GetHcclTypeSize] fail, op:%s(%s)", + op_desc->GetName().c_str(), op_desc->GetType().c_str()); int64_t input_size = 0; int64_t block_size = 0; GE_CHECK_NOTNULL(op_desc->GetInputDescPtr(i)); GE_CHK_STATUS_RET(ge::TensorUtils::GetSize(*op_desc->GetInputDescPtr(i), input_size), - "get size from TensorDesc failed, op : %s, input index : %zu", op_desc->GetName().c_str(), i); + "[Get][Size] from TensorDesc failed, op:%s, input index:%zu", op_desc->GetName().c_str(), i); int64_t shape_size = op_desc->GetInputDescPtr(i)->GetShape().GetShapeSize(); GE_CHK_STATUS_RET(ge::CheckInt64Int32MulOverflow(shape_size, size), - "Product of shape size and size beyond INT64_MAX"); + "[Check][Param] Product of shape size:%ld and size:%d beyond INT64_MAX", shape_size, size); if (kernel_hccl_infos[0].hccl_type == HVDCALLBACKALLGATHER) { block_size = shape_size * size; } else { block_size = (input_size + align_size - 1) / align_size * align_size; } - GE_CHK_BOOL_RET_STATUS(size != 0, PARAM_INVALID, "Size is zero"); - GE_CHK_BOOL_EXEC(block_size % size == 0, return PARAM_INVALID, "block_size:%ld is not divisiable by size:%d.", - block_size, size); + GE_CHK_BOOL_RET_STATUS(size != 0, PARAM_INVALID, "[Check][Param] Size is zero"); + GE_CHK_BOOL_EXEC(block_size % size == 0, return PARAM_INVALID, + "[Check][Param] block_size:%ld is not divisiable by size:%d.", block_size, size); kernel_hccl_infos[i].count = static_cast(block_size / size); } @@ -191,7 +199,8 @@ Status HcomOmeUtil::GetHcclCount(const ge::ConstOpDescPtr &op_desc, Status ret; ret = CheckKernelHcclInfo(op_desc, kernel_hccl_infos); if (ret != SUCCESS) { - GELOGE(PARAM_INVALID, "HcomOmeUtil:: the number of GETaskKernelHcclInfo is invalid."); + GELOGE(PARAM_INVALID, "[Check][KernelHcclInfo] failed, the number of GETaskKernelHcclInfo is invalid, op:%s(%s).", + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return PARAM_INVALID; } GELOGI("GetHcclCount start, node[%s], opType[%s].", op_desc->GetName().c_str(), op_desc->GetType().c_str()); @@ -200,7 +209,7 @@ Status HcomOmeUtil::GetHcclCount(const ge::ConstOpDescPtr &op_desc, ret = GetHcomCount(op_desc, static_cast(kernel_hccl_infos[0].dataType), kernel_hccl_infos[0].hccl_type == HCOMALLGATHER, count); if (ret != SUCCESS) { - GELOGE(ret, "HcomOmeUtil:: Node: %s Optype: %s get the Hcom operator hccl count fail.", + GELOGE(ret, "[Call][GetHcomCount] Node:%s Optype:%s get the Hcom operator hccl count fail.", op_desc->GetName().c_str(), op_desc->GetType().c_str()); return PARAM_INVALID; } @@ -210,7 +219,7 @@ Status HcomOmeUtil::GetHcclCount(const ge::ConstOpDescPtr &op_desc, if (IsHorovodOp(op_desc->GetType())) { ret = GetHorovodCount(op_desc, kernel_hccl_infos); if (ret != SUCCESS) { - GELOGE(PARAM_INVALID, "HcomOmeUtil:: Node: %s Optype: %s get the Horovod hccl operator count fail.", + GELOGE(PARAM_INVALID, "[Call][GetHorovodCount] Node:%s Optype:%s get the Horovod hccl operator count fail.", op_desc->GetName().c_str(), op_desc->GetType().c_str()); return PARAM_INVALID; } @@ -225,11 +234,10 @@ Status HcomOmeUtil::GetHcclOperationType(const ge::ConstOpDescPtr &op_desc, Hccl if (IsHCOMOp(op_desc->GetType())) { std::string hcom_op_type; GE_CHK_BOOL_EXEC(ge::AttrUtils::GetStr(op_desc, HCOM_ATTR_REDUCE_TYPE, hcom_op_type), - REPORT_INNER_ERROR("E19999", "Get Attr:%s in op:%s(%s) fail", - HCOM_ATTR_REDUCE_TYPE.c_str(), + REPORT_INNER_ERROR("E19999", "Get Attr:%s in op:%s(%s) fail", HCOM_ATTR_REDUCE_TYPE.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); return PARAM_INVALID, - "HcomOmeUtil:: Node: %s Optype: %s Get HCOM_ATTR_REDUCE_TYPE fail, not support!", + "[Get][Attr] %s in op:%s(%s) fail", HCOM_ATTR_REDUCE_TYPE.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); if (hcom_op_type == "min") { @@ -244,7 +252,9 @@ Status HcomOmeUtil::GetHcclOperationType(const ge::ConstOpDescPtr &op_desc, Hccl REPORT_INNER_ERROR("E19999", "Attr:%s in Op:%s(%s), hcom_op_type value:%s is not support now, " "check invalid", HCOM_ATTR_REDUCE_TYPE.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str(), hcom_op_type.c_str()); - GELOGE(PARAM_INVALID, "HcomOmeUtil::Get HCOM_ATTR_REDUCE_TYPE fail, [%s] not support!", hcom_op_type.c_str()); + GELOGE(PARAM_INVALID, "[Check][Param] Attr:%s in Op:%s(%s), hcom_op_type value:%s is not support now", + HCOM_ATTR_REDUCE_TYPE.c_str(), op_desc->GetName().c_str(), + op_desc->GetType().c_str(), hcom_op_type.c_str()); return PARAM_INVALID; } } @@ -256,7 +266,7 @@ Status HcomOmeUtil::GetHcclOperationType(const ge::ConstOpDescPtr &op_desc, Hccl ATTR_HOROVOD_ATTR_REDUCE_TYPE.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); return PARAM_INVALID, - "HcomOmeUtil:: Node: %s Optype: %s Get ATTR_HOROVOD_ATTR_REDUCE_TYPE fail, not support!", + "[Get][Attr] %s in op:%s(%s) fail", ATTR_HOROVOD_ATTR_REDUCE_TYPE.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); auto iter = kHorovodRedOpToHcclRedOp.find(static_cast(horovod_op_type)); @@ -264,8 +274,9 @@ Status HcomOmeUtil::GetHcclOperationType(const ge::ConstOpDescPtr &op_desc, Hccl REPORT_INNER_ERROR("E19999", "Attr:%s in Op:%s(%s), horovod_op_type value:%ld is not support now, " "check invalid", ATTR_HOROVOD_ATTR_REDUCE_TYPE.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str(), horovod_op_type); - GELOGE(PARAM_INVALID, "HcomOmeUtil:: Node: %s Optype: %s HcomOpType cann't support! Current HcomOpType : %ld", - op_desc->GetName().c_str(), op_desc->GetType().c_str(), horovod_op_type); + GELOGE(PARAM_INVALID, "[Check][Param] Attr:%s in Op:%s(%s), horovod_op_type value:%ld is not support now", + ATTR_HOROVOD_ATTR_REDUCE_TYPE.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str(), + horovod_op_type); return PARAM_INVALID; } op_type = iter->second; @@ -281,7 +292,7 @@ Status HcomOmeUtil::GetHcclRootId(const ge::ConstOpDescPtr &op_desc, int64_t &ro HCOM_ATTR_ROOT_RANK.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); return PARAM_INVALID, - "HcomOmeUtil::Node %s Optype: %s Get HCOM_ATTR_ROOT_INDEX fail, not support!", + "[Get][Attr] %s in op:%s(%s) fail", HCOM_ATTR_ROOT_RANK.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); return SUCCESS; @@ -296,7 +307,7 @@ Status HcomOmeUtil::GetAllRootId(const ge::ConstOpDescPtr &op_desc, int64_t root_id = 0; Status dmrt = GetHcclRootId(op_desc, root_id); if (dmrt != SUCCESS) { - GELOGE(FAILED, "davinci_model: GetHcomRootId fail! domi error: %u", dmrt); + GELOGE(FAILED, "[Get][HcclRootId] fail! domi error: %u", dmrt); return FAILED; } @@ -324,7 +335,8 @@ Status HcomOmeUtil::CheckKernelHcclInfo(const ge::ConstOpDescPtr &op_desc, REPORT_INNER_ERROR("E19999", "Op:%s(%s) is not hcom op or param kernel_hccl_infos.size:%zu != 1, " "check invalid", op_desc->GetName().c_str(), op_desc->GetType().c_str(), kernel_hccl_infos.size()); - GELOGE(PARAM_INVALID, "HcomOmeUtil:: in Hcom scenario, the number of GETaskKernelHcclInfo is invalid."); + GELOGE(PARAM_INVALID, "[Check][Param] Op:%s(%s) is not hcom op or param kernel_hccl_infos.size:%zu != 1", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), kernel_hccl_infos.size()); return PARAM_INVALID; } @@ -337,7 +349,9 @@ Status HcomOmeUtil::CheckKernelHcclInfo(const ge::ConstOpDescPtr &op_desc, "in op:%s(%s), check invalid", kernel_hccl_infos.size(), op_desc->GetInputsSize(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(PARAM_INVALID, "HcomOmeUtil:: in Horovod scenario, the number of GETaskKernelHcclInfo is invalid."); + GELOGE(PARAM_INVALID, "Param kernel_hccl_infos.size:%zu is empty or not equal to " + "input_desc size:%zu in op:%s(%s)", kernel_hccl_infos.size(), op_desc->GetInputsSize(), + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return PARAM_INVALID; } } @@ -360,7 +374,7 @@ Status HcomOmeUtil::GetHorovodInputs(const ge::ConstOpDescPtr &op_desc, } if (CheckKernelHcclInfo(op_desc, kernel_hccl_infos) != SUCCESS) { - GELOGE(PARAM_INVALID, "HcomOmeUtil:: Node: %s Optype: %s the number of GETaskKernelHcclInfo is invalid.", + GELOGE(PARAM_INVALID, "[Check][KernelHcclInfo] Node:%s Optype:%s the number of GETaskKernelHcclInfo is invalid.", op_desc->GetName().c_str(), op_desc->GetType().c_str()); return PARAM_INVALID; } diff --git a/ge/graph/manager/util/variable_accelerate_ctrl.cc b/ge/graph/manager/util/variable_accelerate_ctrl.cc index 22f9169c..12ec5a49 100644 --- a/ge/graph/manager/util/variable_accelerate_ctrl.cc +++ b/ge/graph/manager/util/variable_accelerate_ctrl.cc @@ -54,7 +54,7 @@ void VarAccelerateCtrl::SetVarChanged(const std::string &var_name) { void VarAccelerateCtrl::AddGraph(uint32_t graph_id, const ComputeGraphPtr &compute_graph) { std::lock_guard lock(mutex_); if (compute_graph == nullptr) { - GELOGE(PARAM_INVALID, "Failed to add graph %u, the compute graph is null", graph_id); + GELOGE(PARAM_INVALID, "[Check][Param] Failed to add graph %u, the compute graph is null", graph_id); return; } auto &var_names = graph_ids_to_var_names_[graph_id]; diff --git a/ge/graph/optimize/graph_optimize.cc b/ge/graph/optimize/graph_optimize.cc index 1dc349a6..3f1a64cc 100644 --- a/ge/graph/optimize/graph_optimize.cc +++ b/ge/graph/optimize/graph_optimize.cc @@ -38,7 +38,7 @@ GraphOptimize::GraphOptimize() void AddNodeInputProperty(ComputeGraphPtr &compute_graph) { if (compute_graph == nullptr) { REPORT_INNER_ERROR("E19999", "Param compute_graph is nullptr, check invalid"); - GELOGE(GE_GRAPH_OPTIMIZE_COMPUTE_GRAPH_NULL, "[AddNodeInputProperty]: compute_graph is nullptr."); + GELOGE(GE_GRAPH_OPTIMIZE_COMPUTE_GRAPH_NULL, "[Check][Param] compute_graph is nullptr."); return; } for (ge::NodePtr &node : compute_graph->GetDirectNode()) { @@ -80,7 +80,7 @@ void AddNodeInputProperty(ComputeGraphPtr &compute_graph) { Status GraphOptimize::OptimizeSubGraph(ComputeGraphPtr &compute_graph, const std::string &engine_name) { if (compute_graph == nullptr) { REPORT_INNER_ERROR("E19999", "Param compute_graph is nullptr, check invalid"); - GELOGE(GE_GRAPH_OPTIMIZE_COMPUTE_GRAPH_NULL, "[OptimizeSubGraph]: compute_graph is nullptr."); + GELOGE(GE_GRAPH_OPTIMIZE_COMPUTE_GRAPH_NULL, "[Check][Param] compute_graph is nullptr."); return GE_GRAPH_OPTIMIZE_COMPUTE_GRAPH_NULL; } @@ -89,8 +89,10 @@ Status GraphOptimize::OptimizeSubGraph(ComputeGraphPtr &compute_graph, const std std::shared_ptr instance_ptr = ge::GELib::GetInstance(); if (instance_ptr == nullptr || !instance_ptr->InitFlag()) { - REPORT_INNER_ERROR("E19999", "Gelib not init before, check invalid"); - GELOGE(GE_CLI_GE_NOT_INITIALIZED, "GraphOptimzer: GE is not initialized"); + REPORT_INNER_ERROR("E19999", "Gelib not init before, check invalid, graph:%s", + compute_graph->GetName().c_str()); + GELOGE(GE_CLI_GE_NOT_INITIALIZED, "[Get][GELib] Gelib not init before, graph:%s", + compute_graph->GetName().c_str()); return GE_CLI_GE_NOT_INITIALIZED; } @@ -111,7 +113,8 @@ Status GraphOptimize::OptimizeSubGraph(ComputeGraphPtr &compute_graph, const std REPORT_INNER_ERROR("E19999", "Call OptimizeFusedGraphAfterGraphSlice failed, ret:%d, engine_name:%s, " "graph_name:%s", ret, engine_name.c_str(), compute_graph->GetName().c_str()); - GELOGE(ret, "[OptimizeSubGraph][OptimizeFusedGraphAfterGraphSlice]: graph optimize failed, ret:%d", ret); + GELOGE(ret, "[Call][OptimizeFusedGraphAfterGraphSlice] failed, ret:%d, engine_name:%s, graph_name:%s", + ret, engine_name.c_str(), compute_graph->GetName().c_str()); return ret; } } @@ -124,7 +127,8 @@ Status GraphOptimize::OptimizeSubGraph(ComputeGraphPtr &compute_graph, const std REPORT_INNER_ERROR("E19999", "Call OptimizeFusedGraph failed, ret:%d, engine_name:%s, " "graph_name:%s", ret, engine_name.c_str(), compute_graph->GetName().c_str()); - GELOGE(ret, "[OptimizeSubGraph][OptimizeFusedGraph]: graph optimize failed, ret:%d", ret); + GELOGE(ret, "[Optimize][FusedGraph] failed, ret:%d, engine_name:%s, graph_name:%s", + ret, engine_name.c_str(), compute_graph->GetName().c_str()); return ret; } } @@ -142,20 +146,22 @@ Status GraphOptimize::OptimizeOriginalGraph(ComputeGraphPtr &compute_graph) { } if (compute_graph == nullptr) { REPORT_INNER_ERROR("E19999", "Param compute_graph is nullptr, check invalid"); - GELOGE(GE_GRAPH_OPTIMIZE_COMPUTE_GRAPH_NULL, "[OptimizeOriginalGraph]: compute_graph is nullptr."); + GELOGE(GE_GRAPH_OPTIMIZE_COMPUTE_GRAPH_NULL, "[Check][Param] compute_graph is nullptr."); return GE_GRAPH_OPTIMIZE_COMPUTE_GRAPH_NULL; } Status ret = SUCCESS; std::shared_ptr instance_ptr = ge::GELib::GetInstance(); if (instance_ptr == nullptr || !instance_ptr->InitFlag()) { - REPORT_INNER_ERROR("E19999", "Gelib not init before, check invalid"); - GELOGE(GE_CLI_GE_NOT_INITIALIZED, "OptimizeOriginalGraph failed."); + REPORT_INNER_ERROR("E19999", "Gelib not init before, check invalid, graph:%s.", + compute_graph->GetName().c_str()); + GELOGE(GE_CLI_GE_NOT_INITIALIZED, "[Get][GELib] Gelib not init before, graph:%s.", + compute_graph->GetName().c_str()); return GE_CLI_GE_NOT_INITIALIZED; } auto graph_optimizer = instance_ptr->OpsKernelManagerObj().GetAllGraphOptimizerObjsByPriority(); - GELOGI("optimize by opskernel in original graph optimize phase. num of graph_optimizer is %lu.", + GELOGI("optimize by opskernel in original graph optimize phase. num of graph_optimizer is %zu.", graph_optimizer.size()); string exclude_core_Type = (core_type_ == kVectorCore) ? kAicoreEngine : kVectorEngine; GELOGD("[OptimizeOriginalGraph]: engine type will exclude: %s", exclude_core_Type.c_str()); @@ -169,7 +175,8 @@ Status GraphOptimize::OptimizeOriginalGraph(ComputeGraphPtr &compute_graph) { REPORT_INNER_ERROR("E19999", "Call OptimizeOriginalGraph failed, ret:%d, engine_name:%s, " "graph_name:%s", ret, iter->first.c_str(), compute_graph->GetName().c_str()); - GELOGE(ret, "[OptimizeOriginalGraph]: graph optimize failed, ret:%d", ret); + GELOGE(ret, "[Optimize][OriginalGraph] failed, ret:%d, engine_name:%s, graph_name:%s", + ret, iter->first.c_str(), compute_graph->GetName().c_str()); return ret; } } @@ -188,13 +195,15 @@ Status GraphOptimize::OptimizeOriginalGraphJudgeInsert(ComputeGraphPtr &compute_ Status ret = SUCCESS; std::shared_ptr instance_ptr = ge::GELib::GetInstance(); if (instance_ptr == nullptr || !instance_ptr->InitFlag()) { - REPORT_INNER_ERROR("E19999", "Gelib not init before, check invalid"); - GELOGE(GE_CLI_GE_NOT_INITIALIZED, "OptimizeOriginalGraph failed."); + REPORT_INNER_ERROR("E19999", "Gelib not init before, check invalid, graph:%s", + compute_graph->GetName().c_str()); + GELOGE(GE_CLI_GE_NOT_INITIALIZED, "[Get][GELib] Gelib not init before, graph:%s", + compute_graph->GetName().c_str()); return GE_CLI_GE_NOT_INITIALIZED; } auto graph_optimizer = instance_ptr->OpsKernelManagerObj().GetAllGraphOptimizerObjsByPriority(); - GELOGI("optimize by opskernel in original graph optimize phase. num of graph_optimizer is %lu.", + GELOGI("optimize by opskernel in judging insert phase. num of graph_optimizer is %zu.", graph_optimizer.size()); string exclude_core_Type = (core_type_ == kVectorCore) ? kAicoreEngine : kVectorEngine; if (graph_optimizer.size() != 0) { @@ -209,7 +218,8 @@ Status GraphOptimize::OptimizeOriginalGraphJudgeInsert(ComputeGraphPtr &compute_ REPORT_INNER_ERROR("E19999", "Call OptimizeOriginalGraphJudgeInsert failed, ret:%d, engine_name:%s, " "graph_name:%s", ret, iter->first.c_str(), compute_graph->GetName().c_str()); - GELOGE(ret, "[OptimizeOriginalGraphJudgeInsert]: graph optimize failed, ret:%d", ret); + GELOGE(ret, "[Call][OptimizeOriginalGraphJudgeInsert] failed, ret:%d, engine_name:%s, graph_name:%s", + ret, iter->first.c_str(), compute_graph->GetName().c_str()); return ret; } } @@ -220,14 +230,16 @@ Status GraphOptimize::OptimizeOriginalGraphJudgeInsert(ComputeGraphPtr &compute_ Status GraphOptimize::OptimizeOriginalGraphForQuantize(ComputeGraphPtr &compute_graph) { if (compute_graph == nullptr) { REPORT_INNER_ERROR("E19999", "Param compute_graph is nullptr, check invalid"); - GELOGE(GE_GRAPH_OPTIMIZE_COMPUTE_GRAPH_NULL, "[OptimizeOriginalGraph]: compute_graph is nullptr."); + GELOGE(GE_GRAPH_OPTIMIZE_COMPUTE_GRAPH_NULL, "[Check][Param] compute_graph is nullptr."); return GE_GRAPH_OPTIMIZE_COMPUTE_GRAPH_NULL; } std::shared_ptr instance_ptr = ge::GELib::GetInstance(); if (instance_ptr == nullptr || !instance_ptr->InitFlag()) { - REPORT_INNER_ERROR("E19999", "Gelib not init before, check invalid"); - GELOGE(GE_CLI_GE_NOT_INITIALIZED, "OptimizeOriginalGraph failed."); + REPORT_INNER_ERROR("E19999", "Gelib not init before, check invalid, graph:%s.", + compute_graph->GetName().c_str()); + GELOGE(GE_CLI_GE_NOT_INITIALIZED, "[Get][Gelib] Gelib not init before, graph:%s.", + compute_graph->GetName().c_str()); return GE_CLI_GE_NOT_INITIALIZED; } @@ -247,7 +259,8 @@ Status GraphOptimize::OptimizeOriginalGraphForQuantize(ComputeGraphPtr &compute_ REPORT_INNER_ERROR("E19999", "Call OptimizeGraphPrepare failed, ret:%d, engine_name:%s, " "graph_name:%s", ret, iter->first.c_str(), compute_graph->GetName().c_str()); - GELOGE(ret, "[OptimizeOriginalGraphForQuantize]: graph optimize failed, ret:%u", ret); + GELOGE(ret, "[Call][OptimizeGraphPrepare] failed, ret:%d, engine_name:%s, graph_name:%s", + ret, iter->first.c_str(), compute_graph->GetName().c_str()); return ret; } } @@ -258,14 +271,16 @@ Status GraphOptimize::OptimizeOriginalGraphForQuantize(ComputeGraphPtr &compute_ Status GraphOptimize::OptimizeGraphBeforeBuildForRts(ComputeGraphPtr &compute_graph) { if (compute_graph == nullptr) { REPORT_INNER_ERROR("E19999", "Param compute_graph is nullptr, check invalid"); - GELOGE(GE_GRAPH_OPTIMIZE_COMPUTE_GRAPH_NULL, "[OptimizeGraphBeforeBuildForRts]: compute_graph is nullptr."); + GELOGE(GE_GRAPH_OPTIMIZE_COMPUTE_GRAPH_NULL, "[Check][Param] compute_graph is nullptr."); return GE_GRAPH_OPTIMIZE_COMPUTE_GRAPH_NULL; } std::shared_ptr instance_ptr = ge::GELib::GetInstance(); if (instance_ptr == nullptr || !instance_ptr->InitFlag()) { - REPORT_INNER_ERROR("E19999", "Gelib not init before, check invalid"); - GELOGE(GE_CLI_GE_NOT_INITIALIZED, "OptimizeGraphBeforeBuildForRts failed."); + REPORT_INNER_ERROR("E19999", "Gelib not init before, check invalid, graph:%s.", + compute_graph->GetName().c_str()); + GELOGE(GE_CLI_GE_NOT_INITIALIZED, "[Get][GELib] Gelib not init before, graph:%s.", + compute_graph->GetName().c_str()); return GE_CLI_GE_NOT_INITIALIZED; } @@ -286,7 +301,48 @@ Status GraphOptimize::OptimizeGraphBeforeBuildForRts(ComputeGraphPtr &compute_gr REPORT_INNER_ERROR("E19999", "Call OptimizeGraphBeforeBuild failed, ret:%d, engine_name:%s, " "graph_name:%s", ret, iter->first.c_str(), compute_graph->GetName().c_str()); - GELOGE(ret, "[OptimizeGraphBeforeBuildForRts]: graph optimize failed, ret:%u", ret); + GELOGE(ret, "[Call][OptimizeGraphBeforeBuild] failed, ret:%d, engine_name:%s, graph_name:%s", + ret, iter->first.c_str(), compute_graph->GetName().c_str()); + return ret; + } + } + } + return ret; +} + +Status GraphOptimize::OptimizeAfterStage1(ComputeGraphPtr &compute_graph) { + GE_CHECK_NOTNULL(compute_graph); + GELOGD("OptimizeAfterStage1 in"); + if (GetContext().GetHostExecFlag()) { + // graph exec on host, no need OptimizeAfterStage1 + return SUCCESS; + } + + Status ret = SUCCESS; + std::shared_ptr instance_ptr = ge::GELib::GetInstance(); + if (instance_ptr == nullptr || !instance_ptr->InitFlag()) { + REPORT_INNER_ERROR("E19999", "Gelib not init before, check invalid"); + GELOGE(GE_CLI_GE_NOT_INITIALIZED, "OptimizeAfterStage1 failed."); + return GE_CLI_GE_NOT_INITIALIZED; + } + + auto graph_optimizer = instance_ptr->OpsKernelManagerObj().GetAllGraphOptimizerObjsByPriority(); + GELOGI("Optimize by ops kernel in after stage1 phase, num of graph_optimizer is %zu.", graph_optimizer.size()); + string exclude_core_type = (core_type_ == kVectorCore) ? kAicoreEngine : kVectorEngine; + if (graph_optimizer.size() != 0) { + for (auto iter = graph_optimizer.begin(); iter != graph_optimizer.end(); ++iter) { + if (iter->first == exclude_core_type) { + GELOGI("[OptimizeAfterStage1]: engine type will exclude:%s.", exclude_core_type.c_str()); + continue; + } +#ifndef ONLY_COMPILE_OPEN_SRC + GELOGI("Begin to optimize graph after stage1 by engine %s.", iter->first.c_str()); + ret = (iter->second)->OptimizeAfterStage1(*compute_graph); +#endif + if (ret != SUCCESS) { + REPORT_INNER_ERROR("E19999", "Call OptimizeAfterStage1 failed, ret:%d, engine_name:%s, " + "graph_name:%s.", ret, iter->first.c_str(), compute_graph->GetName().c_str()); + GELOGE(ret, "[OptimizeAfterStage1]: graph optimize failed, ret:%d.", ret); return ret; } } @@ -373,14 +429,16 @@ Status GraphOptimize::IdentifyReference(ComputeGraphPtr &compute_graph) { Status GraphOptimize::OptimizeWholeGraph(ComputeGraphPtr &compute_graph) { if (compute_graph == nullptr) { REPORT_INNER_ERROR("E19999", "Param compute_graph is nullptr, check invalid"); - GELOGE(GE_GRAPH_OPTIMIZE_COMPUTE_GRAPH_NULL, "[OptimizeWholeGraph]: compute_graph is nullptr."); + GELOGE(GE_GRAPH_OPTIMIZE_COMPUTE_GRAPH_NULL, "[Check][Param] compute_graph is nullptr."); return GE_GRAPH_OPTIMIZE_COMPUTE_GRAPH_NULL; } std::shared_ptr instance_ptr = ge::GELib::GetInstance(); if (instance_ptr == nullptr || !instance_ptr->InitFlag()) { - REPORT_INNER_ERROR("E19999", "Gelib not init before, check invalid"); - GELOGE(GE_CLI_GE_NOT_INITIALIZED, "OptimizeWholeGraph failed."); + REPORT_INNER_ERROR("E19999", "Gelib not init before, check invalid, graph:%s.", + compute_graph->GetName().c_str()); + GELOGE(GE_CLI_GE_NOT_INITIALIZED, "[Get][GELib] Gelib not init before, graph:%s.", + compute_graph->GetName().c_str()); return GE_CLI_GE_NOT_INITIALIZED; } @@ -401,7 +459,8 @@ Status GraphOptimize::OptimizeWholeGraph(ComputeGraphPtr &compute_graph) { REPORT_INNER_ERROR("E19999", "Call OptimizeWholeGraph failed, ret:%d, engine_name:%s, " "graph_name:%s", ret, iter.first.c_str(), compute_graph->GetName().c_str()); - GELOGE(ret, "[OptimizeWholeGraph]: graph optimize failed, ret:%u", ret); + GELOGE(ret, "[Call][OptimizeWholeGraph] failed, ret:%d, engine_name:%s, graph_name:%s", + ret, iter.first.c_str(), compute_graph->GetName().c_str()); return ret; } } diff --git a/ge/graph/optimize/graph_optimize.h b/ge/graph/optimize/graph_optimize.h index 3a1960f7..702b7e33 100755 --- a/ge/graph/optimize/graph_optimize.h +++ b/ge/graph/optimize/graph_optimize.h @@ -58,6 +58,9 @@ class GraphOptimize { // for rts optimize before build to add attr and insert memcpy op Status OptimizeGraphBeforeBuildForRts(ComputeGraphPtr &compute_graph); + // optimize whole graph, using after stage1 + Status OptimizeAfterStage1(ComputeGraphPtr &graph); + // set options Status SetOptions(const GraphManagerOptions &options); diff --git a/ge/graph/optimize/mem_rw_conflict_optimize.cc b/ge/graph/optimize/mem_rw_conflict_optimize.cc index 077ed110..6ffd099d 100644 --- a/ge/graph/optimize/mem_rw_conflict_optimize.cc +++ b/ge/graph/optimize/mem_rw_conflict_optimize.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -22,6 +22,7 @@ #include "graph/optimize/graph_optimize.h" #include "graph/utils/graph_utils.h" #include "graph/utils/node_utils.h" +#include "graph/utils/op_desc_utils.h" namespace { using namespace ge; @@ -32,12 +33,14 @@ const int kCaseReadOnly = 0; const int kCaseScopeWriteable = 2; const int kCaseWriteable = 3; const int kCaseInvalidRWType = 5; +// attr _input_mutable = true means node will modify its input in runtime +const char *const kModifyInput = "_input_mutable"; // rw type of input. enum class InputRWType { kReadOnly, // Normal op input only read kWriteable, // Op like Assign/ApplyMomentum - kScopeWriteable, // Op like hcom_allreduce, it will modify input ,but not expect take effect on pre ouput + kScopeWriteable, // Op like hcom_allreduce/while, it will modify input ,but not expect take effect on pre ouput kInvalidRWType }; // rw type of output @@ -154,38 +157,31 @@ bool IsSubgraphOutputNode(const NodePtr &node) { return true; } -NodePtr CreateIdentityAfterSrcNode(const Node &src_node, int out_anchor_idx) { +NodePtr AddIdentityToGraph(const Node &src_node, int out_anchor_idx) { if (src_node.GetOpDesc() == nullptr) { + REPORT_INNER_ERROR("E19999", "Param src_node is invalid, which has no opdesc"); + GELOGE(GRAPH_PARAM_INVALID, "[Get][OpDesc] failed, Param src_node opdesc is nullptr."); return nullptr; } static std::atomic_long identity_num(0); auto next_num = identity_num.fetch_add(1); // 1. create new identity op desc string identity_name = src_node.GetName() + "_" + IDENTITY + std::to_string(next_num); - auto identity_opdesc = MakeShared(identity_name, IDENTITY); - if (identity_opdesc == nullptr) { - GELOGE(OUT_OF_MEMORY, "Failed to insert identity node, name %s", identity_name.c_str()); - return nullptr; - } + OpDescBuilder op_desc_builder(identity_name, IDENTITY); auto data_desc = src_node.GetOpDesc()->GetOutputDesc(out_anchor_idx); - // 2. add input_desc & output_desc for new identity - Status ret = identity_opdesc->AddInputDesc("x", data_desc); - if (ret != SUCCESS) { - GELOGE(ret, "Add Input desc failed for new identity %s.", identity_name.c_str()); - return nullptr; - } - ret = identity_opdesc->AddOutputDesc("y", data_desc); - if (ret != SUCCESS) { - GELOGE(ret, "Add Output desc failed for new Identity %s.", identity_name.c_str()); - return nullptr; - } + auto identity_op_desc = op_desc_builder.AddInput("x", data_desc) + .AddOutput("y", data_desc) + .Build(); + GELOGI("Insert new Identity node %s.", identity_name.c_str()); auto graph = src_node.GetOwnerComputeGraph(); if (graph == nullptr) { - GELOGE(GRAPH_PARAM_INVALID, "Node %s owner compute graph is null.", src_node.GetName().c_str()); + REPORT_INNER_ERROR("E19999", "Node %s owner compute graph is nullptr.", src_node.GetName().c_str()); + GELOGE(GRAPH_PARAM_INVALID, "[Get][OwnerComputeGraph] failed, as Node %s owner compute graph is nullptr.", + src_node.GetName().c_str()); return nullptr; } - return graph->AddNode(identity_opdesc); + return graph->AddNode(identity_op_desc); } OutputRWType GetOutputRWTypeByIndex(const Node &node, uint32_t index) { @@ -274,8 +270,6 @@ InputRWType GetInputRWTypeByIndex(const Node &node, uint32_t index) { // single node without sub graph return GetSingleNodeInputRWTypeByIndex(node, index); } else { - // node with sub graph - std::set node_rw_type_set; auto data_node_vec = NodeUtils::GetSubgraphDataNodesByIndex(node, index); // get all input data node in subgraph std::set anchor_rw_type_set; @@ -345,12 +339,24 @@ Status MarkRWTypeForSubgraph(const ComputeGraphPtr &sub_graph) { auto parent_node = sub_graph->GetParentNode(); if (pre_output_rw_type == OutputRWType::kWriteable && parent_node->GetType() != PARTITIONEDCALL) { // insert identity - auto identity_node = CreateIdentityAfterSrcNode(*pre_node, pre_out_anchor->GetIdx()); + auto identity_node = AddIdentityToGraph(*pre_node, pre_out_anchor->GetIdx()); GE_CHECK_NOTNULL(identity_node); - auto ret = GraphUtils::InsertNodeBetweenDataAnchors(pre_out_anchor, in_data_anchor, identity_node); - if (ret != SUCCESS) { - GELOGE(ret, "Fail to insert identity"); - return ret; + if (GraphUtils::InsertNodeAfter(pre_out_anchor, {in_data_anchor}, identity_node) != GRAPH_SUCCESS) { + REPORT_CALL_ERROR("E19999", "Insert Identity node %s(%s) between %s(%s) -> %s(%s) failed.", + identity_node->GetName().c_str(), + identity_node->GetType().c_str(), + pre_node->GetName().c_str(), + pre_node->GetType().c_str(), + node->GetName().c_str(), + node->GetType().c_str()); + GELOGE(FAILED, "[Insert][IdentityNode] %s(%s) between %s(%s) -> %s(%s) failed.", + identity_node->GetName().c_str(), + identity_node->GetType().c_str(), + pre_node->GetName().c_str(), + pre_node->GetType().c_str(), + node->GetName().c_str(), + node->GetType().c_str()); + return FAILED; } GELOGI("InsertNode %s between %s and %s successfully.", identity_node->GetName().c_str(), pre_node->GetName().c_str(), node->GetName().c_str()); @@ -484,12 +490,14 @@ Status RemoveNoUseIdentity(const NodePtr &node) { GELOGI("No need insert Identity. Node %s need to remove.", node->GetName().c_str()); auto ret = GraphUtils::IsolateNode(node, {0}); if (ret != SUCCESS) { - GELOGE(ret, "Fail to isolate node %s.", node->GetName().c_str()); + REPORT_CALL_ERROR("E19999", "Isolate Node:%s failed", node->GetName().c_str()); + GELOGE(ret, "[Isolate][Node] %s failed.", node->GetName().c_str()); return ret; } ret = GraphUtils::RemoveNodeWithoutRelink(node->GetOwnerComputeGraph(), node); if (ret != SUCCESS) { - GELOGE(ret, "Fail to isolate node %s.", node->GetName().c_str()); + REPORT_CALL_ERROR("E19999", "Call RemoveNodeWithoutRelink failed, node:%s", node->GetName().c_str()); + GELOGE(ret, "[Call][RemoveNodeWithoutRelink] failed for node %s.", node->GetName().c_str()); return ret; } GELOGI("Pre node is %s and %dth output rw type is %s. Isolate and remove Identity node %s.", @@ -505,34 +513,24 @@ Status SplitIdentityAlongAnchor(const OutDataAnchorPtr &out_data_anchor, const I auto peer_in_data_node = peer_in_data_anchor->GetOwnerNode(); GE_CHECK_NOTNULL(peer_in_data_node); auto input_rw_type = GetInputRWTypeByIndex(*peer_in_data_node, peer_in_data_anchor->GetIdx()); - auto ret = out_data_anchor->Unlink(peer_in_data_anchor); auto old_identity = out_data_anchor->GetOwnerNode(); - if (ret != SUCCESS) { - GELOGE(ret, "Failed to unlink from %s %dth out to %s.", old_identity->GetName().c_str(), out_data_anchor->GetIdx(), - peer_in_data_anchor->GetOwnerNode()->GetName().c_str()); - return ret; - } if (input_rw_type == InputRWType::kScopeWriteable || input_rw_type == InputRWType::kWriteable) { - auto new_identity = CreateIdentityAfterSrcNode(*pre_node, pre_out_data_anchor->GetIdx()); + auto new_identity = AddIdentityToGraph(*pre_node, pre_out_data_anchor->GetIdx()); GE_CHECK_NOTNULL(new_identity); - if (GraphUtils::AddEdge(pre_out_data_anchor, new_identity->GetInDataAnchor(kIdentityAnchorIndex)) != SUCCESS - || GraphUtils::AddEdge(new_identity->GetOutDataAnchor(kIdentityAnchorIndex), peer_in_data_anchor) != SUCCESS) { - GELOGE(INTERNAL_ERROR, "Failed to insert Identity between node %s and %s", - pre_out_data_anchor->GetOwnerNode()->GetName().c_str(), - peer_in_data_anchor->GetOwnerNode()->GetName().c_str()); - return INTERNAL_ERROR; - } - - // 2. copy in-control-edge from dst to Identity - if (GraphUtils::CopyInCtrlEdges(peer_in_data_node, new_identity) != SUCCESS) { - GELOGE(INTERNAL_ERROR, "Failed to copy in_control edges from node %s to %s", peer_in_data_node->GetName().c_str(), - new_identity->GetName().c_str()); - return INTERNAL_ERROR; + auto ret = GraphUtils::InsertNodeBefore(peer_in_data_anchor, new_identity, kIdentityAnchorIndex, + kIdentityAnchorIndex); + if (ret != SUCCESS) { + GELOGE(ret, "[Insert][Identity] %s before %s %dth input failed.", + new_identity->GetName().c_str(), + peer_in_data_anchor->GetOwnerNode()->GetName().c_str(), + peer_in_data_anchor->GetIdx()); + return ret; } GELOGI("Node %s intput rw type is %s. Insert Identity between %s and %s.", peer_in_data_node->GetName().c_str(), InputRWTypeToSerialString(input_rw_type).c_str(), pre_out_data_anchor->GetOwnerNode()->GetName().c_str(), peer_in_data_anchor->GetOwnerNode()->GetName().c_str()); } else { + (void) out_data_anchor->Unlink(peer_in_data_anchor); // copy control edge to pre and peer node if (GraphUtils::CopyInCtrlEdges(old_identity, peer_in_data_node) != SUCCESS || GraphUtils::CopyOutCtrlEdges(old_identity, pre_node) != SUCCESS) { @@ -568,7 +566,8 @@ Status SplitIdentity(const NodePtr &node) { for (const auto &peer_in_data_anchor : out_data_anchor->GetPeerInDataAnchors()) { Status ret = SplitIdentityAlongAnchor(out_data_anchor, peer_in_data_anchor, pre_out_data_anchor, pre_node); if (ret != SUCCESS) { - GELOGE(ret, "Split identity node along anchor failed."); + GELOGE(ret, "[Call][SplitIdentityAlongAnchor] failed, ret:%d, node:%s, pre_node:%s.", + ret, node->GetName().c_str(), pre_node->GetName().c_str()); return ret; } } @@ -576,12 +575,15 @@ Status SplitIdentity(const NodePtr &node) { if (node->GetOutDataNodesSize() == 0) { Status ret = GraphUtils::IsolateNode(node, {}); if (ret != SUCCESS) { - GELOGE(FAILED, "IsolateAndDelete identity node %s.", node->GetName().c_str()); + REPORT_CALL_ERROR("E19999", "IsolateNode %s failed, ret:%d", node->GetName().c_str(), ret); + GELOGE(FAILED, "[Isolate][Node] %s failed, ret:%d", node->GetName().c_str(), ret); return FAILED; } ret = GraphUtils::RemoveNodeWithoutRelink(node->GetOwnerComputeGraph(), node); if (ret != SUCCESS) { - GELOGE(FAILED, "IsolateAndDelete identity node %s.", node->GetName().c_str()); + REPORT_CALL_ERROR("E19999", "Call RemoveNodeWithoutRelink failed, node:%s", node->GetName().c_str()); + GELOGE(FAILED, "[Call][RemoveNodeWithoutRelink] IsolateAndDelete identity node %s failed.", + node->GetName().c_str()); return FAILED; } GELOGI("IsolateAndDelete identity node %s.", node->GetName().c_str()); @@ -613,16 +615,14 @@ Status InsertIdentityAsNeeded(const NodePtr &node) { GELOGD("No need insert Identity."); continue; case INSERT_IDENTITY: - auto identity_node = CreateIdentityAfterSrcNode(*node, out_data_anchor->GetIdx()); - if (identity_node == nullptr) { - GELOGE(FAILED, "Create identity node failed."); - return FAILED; - } - auto ret = GraphUtils::InsertNodeBetweenDataAnchors(out_data_anchor, peer_in_data_anchor, identity_node); - if (ret != GRAPH_SUCCESS) { - GELOGE(INTERNAL_ERROR, "Failed to insert reshape between node %s and %s", node->GetName().c_str(), - peer_in_node->GetName().c_str()); - return INTERNAL_ERROR; + auto identity_node = AddIdentityToGraph(*node, out_data_anchor->GetIdx()); + GE_CHECK_NOTNULL(identity_node); + auto ret = GraphUtils::InsertNodeBefore(peer_in_data_anchor, identity_node, kIdentityAnchorIndex, + kIdentityAnchorIndex); + if (ret != SUCCESS) { + GELOGE(ret, "[Insert][Node] %s before %s %dth input failed.", identity_node->GetName().c_str(), + peer_in_data_anchor->GetOwnerNode()->GetName().c_str(), peer_in_data_anchor->GetIdx()); + return ret; } GELOGI("Insert Identity between %s and %s to handle memory conflict.", node->GetName().c_str(), peer_in_node->GetName().c_str()); @@ -633,28 +633,35 @@ Status InsertIdentityAsNeeded(const NodePtr &node) { return SUCCESS; } Status HandleAllreduceDuplicateInput(ComputeGraphPtr &compute_graph) { - for (const auto &node : compute_graph->GetDirectNode()) { - if (node->GetType() == HCOMALLREDUCE) { - std::set pre_out_anchor_set; - for (const auto &in_data_anchor : node->GetAllInDataAnchors()) { - auto pre_out_anchor = in_data_anchor->GetPeerOutAnchor(); - GE_CHECK_NOTNULL(pre_out_anchor); - if (pre_out_anchor_set.find(pre_out_anchor) == pre_out_anchor_set.end()) { - pre_out_anchor_set.emplace(pre_out_anchor); - continue; - } - // need insert identity - auto pre_node = pre_out_anchor->GetOwnerNode(); - auto identity_node = CreateIdentityAfterSrcNode(*pre_node, pre_out_anchor->GetIdx()); - GE_CHECK_NOTNULL(identity_node); - auto ret = GraphUtils::InsertNodeBetweenDataAnchors(pre_out_anchor, in_data_anchor, identity_node); - GE_CHK_STATUS_RET(ret, "Fail to insert identity."); - GELOGI("InsertNode %s between %s and %s successfully.", identity_node->GetName().c_str(), - pre_node->GetName().c_str(), node->GetName().c_str()); - } - } - } - return SUCCESS; + for (const auto &node : compute_graph->GetDirectNode()) { + bool mutable_input_flag = false; + (void)AttrUtils::GetBool(node->GetOpDesc(), kModifyInput, mutable_input_flag); + if (!mutable_input_flag) { + continue; + } + std::set pre_out_anchor_set; + for (const auto &in_data_anchor : node->GetAllInDataAnchors()) { + auto pre_out_anchor = in_data_anchor->GetPeerOutAnchor(); + GE_CHECK_NOTNULL(pre_out_anchor); + if (pre_out_anchor_set.insert(pre_out_anchor).second) { + continue; + } + // need insert identity + auto pre_node = pre_out_anchor->GetOwnerNode(); + auto identity_node = AddIdentityToGraph(*pre_node, pre_out_anchor->GetIdx()); + GE_CHECK_NOTNULL(identity_node); + auto ret = + GraphUtils::InsertNodeBefore(in_data_anchor, identity_node, kIdentityAnchorIndex, kIdentityAnchorIndex); + if (ret != SUCCESS) { + GELOGE(ret, "[Insert][Node] %s before %s %dth input failed.", identity_node->GetName().c_str(), + node->GetName().c_str(), in_data_anchor->GetIdx()); + return ret; + } + GELOGI("InsertNode %s between %s and %s successfully.", identity_node->GetName().c_str(), + pre_node->GetName().c_str(), node->GetName().c_str()); + } + } + return SUCCESS; } } // namespace @@ -669,7 +676,7 @@ Status GraphOptimize::CheckRWConflict(ComputeGraphPtr &compute_graph, bool &has_ // 1.loop all subgraph, mark rw type from inside to outside Status ret = MarkRWTypeForAllSubgraph(sub_graph_vec); if (ret != SUCCESS) { - GELOGE(ret, "Fail to mark rw type for subgraph."); + GELOGE(ret, "[Call][MarkRWTypeForAllSubgraph] failed for %s.", compute_graph->GetName().c_str()); return ret; } has_conflict = false; @@ -725,7 +732,7 @@ Status GraphOptimize::HandleMemoryRWConflict(ComputeGraphPtr &compute_graph) { // 1.loop all subgraph, mark rw type from inside to outside Status ret = MarkRWTypeForAllSubgraph(sub_graph_vec); if (ret != SUCCESS) { - GELOGE(ret, "Fail to mark rw type for subgraph."); + GELOGE(ret, "[Call][MarkRWTypeForAllSubgraph] failed for %s.", compute_graph->GetName().c_str()); return ret; } // 2.loop all node, including node in subgraph and handle memory rw conflict @@ -752,20 +759,20 @@ Status GraphOptimize::HandleMemoryRWConflict(ComputeGraphPtr &compute_graph) { // split identity ret = SplitIdentity(node); if (ret != SUCCESS) { - GELOGE(ret, "Fail to split identity node %s.", node->GetName().c_str()); + GELOGE(ret, "[Split][Identity] %s failed.", node->GetName().c_str()); return ret; } // remove no use identity ret = RemoveNoUseIdentity(node); if (ret != SUCCESS) { - GELOGE(ret, "Fail to remove useless identity node %s.", node->GetName().c_str()); + GELOGE(ret, "[Remove][Identity] %s failed.", node->GetName().c_str()); return ret; } } // insert Identity ret = InsertIdentityAsNeeded(node); if (ret != SUCCESS) { - GELOGE(ret, "Fail to insert Identity node."); + GELOGE(ret, "[Insert][Identity] %s failed.", node->GetName().c_str()); return ret; } } diff --git a/ge/graph/optimize/summary_optimize.cc b/ge/graph/optimize/summary_optimize.cc index 077ab1b0..d3c02d3e 100644 --- a/ge/graph/optimize/summary_optimize.cc +++ b/ge/graph/optimize/summary_optimize.cc @@ -32,16 +32,19 @@ namespace ge { Status GraphOptimize::HandleSummaryOp(ComputeGraphPtr &compute_graph) { GELOGI("[HandleSummaryOp] HandleSummaryOp start!"); if (summary_output_indexes_.size() >= kMaxMapSize) { - GELOGE(FAILED, "Map size out of range."); + REPORT_INNER_ERROR("E19999", "Map size:%zu out of range:%d, check invalid.", + summary_output_indexes_.size(), kMaxMapSize); + GELOGE(FAILED, "[Check][Param] Map size:%zu out of range:%d.", summary_output_indexes_.size(), kMaxMapSize); return FAILED; } - if (summary_output_indexes_.find(compute_graph->GetGraphID()) != summary_output_indexes_.end()) { - return SUCCESS; - } if (compute_graph == nullptr) { - GELOGE(GE_GRAPH_PARAM_NULLPTR, "compute_graph is nullptr."); + REPORT_INNER_ERROR("E19999", "Param compute_graph is nullptr, check invalid."); + GELOGE(GE_GRAPH_PARAM_NULLPTR, "[Check][Param] compute_graph is nullptr."); return GE_GRAPH_PARAM_NULLPTR; } + if (summary_output_indexes_.find(compute_graph->GetGraphID()) != summary_output_indexes_.end()) { + return SUCCESS; + } vector del_nodes; vector front_nodes; vector out_index; @@ -56,12 +59,18 @@ Status GraphOptimize::HandleSummaryOp(ComputeGraphPtr &compute_graph) { compute_graph->SetSummaryFlag(true); auto in = node_ptr->GetInDataAnchor(0); if (in == nullptr) { - GELOGE(GE_GRAPH_PARAM_NULLPTR, "in is nullptr."); + REPORT_INNER_ERROR("E19999", "In data anchor(index:0) of node:%s is nullptr", node_ptr->GetName().c_str()); + GELOGE(GE_GRAPH_PARAM_NULLPTR, "[Get][InDataAnchor] of node:%s is nullptr, index:0", + node_ptr->GetName().c_str()); return GE_GRAPH_PARAM_NULLPTR; } auto peerin = in->GetPeerOutAnchor(); - GE_IF_BOOL_EXEC(peerin == nullptr, GELOGE(GE_GRAPH_PARAM_NULLPTR, "peerin is nullptr."); + GE_IF_BOOL_EXEC(peerin == nullptr, + REPORT_INNER_ERROR("E19999", "peer out anchor is nullptr, node:%s, in anchor index:0", + node_ptr->GetName().c_str()); + GELOGE(GE_GRAPH_PARAM_NULLPTR, "[Get][PeerOutAnchor] of node:%s is nullptr, in anchor index:0", + node_ptr->GetName().c_str()); return GE_GRAPH_PARAM_NULLPTR); auto ret = GraphUtils::RemoveEdge(peerin, in); @@ -94,7 +103,10 @@ Status GraphOptimize::HandleSummaryOp(ComputeGraphPtr &compute_graph) { for (auto &node_ptr : del_nodes) { auto ret = GraphUtils::RemoveNodeWithoutRelink(compute_graph, node_ptr); if (ret != SUCCESS) { - GELOGE(ret, "GraphUtils::RemoveNodeWithoutRelink failed."); + REPORT_CALL_ERROR("E19999", "Call RemoveNodeWithoutRelink failed, node:%s, graph:%s", + node_ptr->GetName().c_str(), compute_graph->GetName().c_str()); + GELOGE(ret, "[Call][RemoveNodeWithoutRelink] failed, node:%s, graph:%s", + node_ptr->GetName().c_str(), compute_graph->GetName().c_str()); return ret; } // update Target list diff --git a/ge/graph/partition/dynamic_shape_partition.cc b/ge/graph/partition/dynamic_shape_partition.cc index 516d06d1..055b2aa4 100755 --- a/ge/graph/partition/dynamic_shape_partition.cc +++ b/ge/graph/partition/dynamic_shape_partition.cc @@ -36,6 +36,7 @@ #define REQUIRE(cond, ...) \ do { \ if (!(cond)) { \ + REPORT_INNER_ERROR("E19999", __VA_ARGS__); \ GELOGE(FAILED, "[Dynamic shape partition]" __VA_ARGS__); \ return FAILED; \ } \ @@ -46,11 +47,6 @@ #define REQUIRE_GRAPH_SUCCESS(cond, ...) REQUIRE(((cond) == GRAPH_SUCCESS), __VA_ARGS__) namespace ge { -namespace { -const std::set kControlFlowOps{ - STREAMACTIVE, STREAMSWITCH, STREAMMERGE, ENTER, REFENTER, LOOPCOND, NEXTITERATION, REFNEXTITERATION, EXIT, REFEXIT -}; -} using Cluster = DynamicShapePartitioner::Cluster; using ClusterPtr = std::shared_ptr; @@ -68,32 +64,32 @@ static bool IsSingleOpScene(const ComputeGraphPtr &root_graph) { } Status DynamicShapePartitioner::Partition() { - REQUIRE_NOT_NULL(root_graph_, "Graph is nullptr."); + REQUIRE_NOT_NULL(root_graph_, "[Check][Param] Graph is nullptr."); if (IsSingleOpScene(root_graph_)) { GELOGD("Skip dynamic shape partition as in single op scene."); REQUIRE(AttrUtils::SetBool(*root_graph_, ATTR_NAME_DYNAMIC_SHAPE_PARTITIONED, false), - "Failed set dynamic shape partitioned flag on root graph."); + "[Set][Attr] dynamic shape partitioned flag on root graph:%s failed.", root_graph_->GetName().c_str()); return SUCCESS; } GELOGD("Start dynamic shape partition graph %s.", root_graph_->GetName().c_str()); - REQUIRE_SUCCESS(MarkUnknownShapeNodes(), "Failed mark unknown shape nodes, root grah name:%s.", + REQUIRE_SUCCESS(MarkUnknownShapeNodes(), "[Call][MarkUnknownShapeNodes] failed, root grah name:%s.", root_graph_->GetName().c_str()); if (unknown_shape_nodes_.empty()) { GELOGD("Skip dynamic shape partition of graph %s as all nodes are known shape.", root_graph_->GetName().c_str()); REQUIRE(AttrUtils::SetBool(*root_graph_, ATTR_NAME_DYNAMIC_SHAPE_PARTITIONED, false), - "Failed set dynamic shape partitioned flag on root graph %s.", root_graph_->GetName().c_str()); + "[Set][Attr] dynamic shape partitioned flag on root graph %s failed.", root_graph_->GetName().c_str()); return SUCCESS; } REQUIRE(AttrUtils::SetBool(*root_graph_, ATTR_NAME_DYNAMIC_SHAPE_PARTITIONED, true), - "Failed set dynamic shape partitioned flag on root graph %s.", root_graph_->GetName().c_str()); - REQUIRE_SUCCESS(CtrlEdgeTransfer(), "Failed do ctrl edge transfer!"); + "[Set][Attr] dynamic shape partitioned flag on root graph %s failed.", root_graph_->GetName().c_str()); + REQUIRE_SUCCESS(CtrlEdgeTransfer(), "[Call][CtrlEdgeTransfer] failed, graph:%s.", root_graph_->GetName().c_str()); DumpGraph("_Before_DSP"); auto status = PartitionImpl(); GELOGD("%s.", DebugString().c_str()); if (status != SUCCESS) { - GELOGE(status, "Failed dynamic shape partition graph: %s, status:\n %s", root_graph_->GetName().c_str(), - DebugString().c_str()); + GELOGE(status, "[Call][PartitionImpl] Failed dynamic shape partition graph:%s, ret:%s", + root_graph_->GetName().c_str(), DebugString().c_str()); } DumpGraph("_After_DSP"); GELOGD("Finish dynamic shape partition graph %s.", root_graph_->GetName().c_str()); @@ -128,13 +124,19 @@ Status DynamicShapePartitioner::CtrlEdgeTransfer() { for (auto &in_control_node : n->GetInControlNodes()) { GE_CHECK_NOTNULL(in_control_node); GE_CHK_STATUS_RET(ge::GraphUtils::RemoveEdge(in_control_node->GetOutControlAnchor(), - n->GetInControlAnchor()), "remove edge failed"); + n->GetInControlAnchor()), + "[Remove][Edge] between %s and %s failed", + in_control_node->GetOutControlAnchor()->GetOwnerNode()->GetName().c_str(), + n->GetName().c_str()); for (auto &out_node : n->GetOutNodes()) { if (out_node == nullptr) { continue; } GE_CHK_STATUS_RET(ge::GraphUtils::AddEdge(in_control_node->GetOutControlAnchor(), - out_node->GetInControlAnchor()), "add edge failed."); + out_node->GetInControlAnchor()), + "[Add][Edge] between %s and %s failed.", + in_control_node->GetOutControlAnchor()->GetOwnerNode()->GetName().c_str(), + out_node->GetName().c_str()); } } } @@ -146,13 +148,16 @@ Status DynamicShapePartitioner::CtrlEdgeTransfer() { } Status DynamicShapePartitioner::PartitionImpl() { - REQUIRE_SUCCESS(root_graph_->TopologicalSorting(), "Graph topological sort failed."); - REQUIRE_SUCCESS(InitClusters(), "Failed init cluster nodes."); - REQUIRE_SUCCESS(MergeClusters(), "Failed merge clusters."); + REQUIRE_SUCCESS(root_graph_->TopologicalSorting(), + "[Call][TopologicalSorting] failed, graph:%s.", root_graph_->GetName().c_str()); + REQUIRE_SUCCESS(InitClusters(), "[Init][Clusters] failed, graph:%s.", root_graph_->GetName().c_str()); + REQUIRE_SUCCESS(MergeClusters(), "[Merge][Clusters] failed, graph:%s.", root_graph_->GetName().c_str()); PruneUniqueClusters(); - REQUIRE_SUCCESS(BuildPartitionFrame(), "Failed build cluster partition frame."); - REQUIRE_SUCCESS(CombinePartitionFrame(), "Failed combine cluster partition frame."); - REQUIRE_SUCCESS(BuildPartitionSubgraph(), "Failed build cluster partition subgraph."); + REQUIRE_SUCCESS(BuildPartitionFrame(), "[Build][PartitionFrame] failed, graph:%s.", root_graph_->GetName().c_str()); + REQUIRE_SUCCESS(CombinePartitionFrame(), + "[Combine][PartitionFrame] failed, graph:%s.", root_graph_->GetName().c_str()); + REQUIRE_SUCCESS(BuildPartitionSubgraph(), + "[Build][PartitionSubgraph] failed, graph:%s.", root_graph_->GetName().c_str()); return SUCCESS; } @@ -174,21 +179,21 @@ void DynamicShapePartitioner::PruneUniqueClusters() { Status DynamicShapePartitioner::BuildPartitionFrame() { for (const auto &cluster : sorted_unique_clusters_) { - REQUIRE_SUCCESS(cluster->BuildFrame(), "Failed build frame of cluster[%lu].", cluster->Id()); + REQUIRE_SUCCESS(cluster->BuildFrame(), "[Build][Frame] of cluster[%lu] failed.", cluster->Id()); } return SUCCESS; } Status DynamicShapePartitioner::CombinePartitionFrame() { for (const auto &cluster : sorted_unique_clusters_) { - REQUIRE_SUCCESS(cluster->CombinePartitionFrame(), "Failed combine frame of cluster[%lu].", cluster->Id()); + REQUIRE_SUCCESS(cluster->CombinePartitionFrame(), "[Combine][Frame] of cluster[%lu] failed.", cluster->Id()); } return SUCCESS; } Status DynamicShapePartitioner::BuildPartitionSubgraph() { for (const auto &cluster : sorted_unique_clusters_) { - REQUIRE_SUCCESS(cluster->BuildPartitionSubgraph(), "Failed build subgraph of cluster[%lu].", cluster->Id()); + REQUIRE_SUCCESS(cluster->BuildPartitionSubgraph(), "[Build][SubGraph] of cluster[%lu] failed.", cluster->Id()); } return SUCCESS; } @@ -250,8 +255,8 @@ void DynamicShapePartitioner::ClearResource() { Status DynamicShapePartitioner::MarkUnknownShapeNodes() { for (auto &node : root_graph_->GetDirectNode()) { - REQUIRE_SUCCESS(CollectSpreadUnknownShapeNodes(node), "Failed collect spread unknown shape nodes %s.", - node->GetName().c_str()); + REQUIRE_SUCCESS(CollectSpreadUnknownShapeNodes(node), + "[Call][CollectSpreadUnknownShapeNodes] for node:%s failed.", node->GetName().c_str()); } return SUCCESS; } @@ -262,7 +267,7 @@ Status DynamicShapePartitioner::InitClusters() { for (const auto &node : graph->GetDirectNode()) { Cluster::Type type = Cluster::DATA; bool is_input = ((node->GetType() == CONSTANT) || (node->GetType() == CONSTANTOP)) && node->GetInNodes().empty(); - REQUIRE_NOT_NULL(node->GetOpDesc(), "op_desc is null"); + REQUIRE_NOT_NULL(node->GetOpDesc(), "[Get][OpDesc] op_desc is null, graph:%s", graph->GetName().c_str()); if (node->GetType() == DATA) { type = Cluster::DATA; } else if (is_input) { @@ -277,11 +282,19 @@ Status DynamicShapePartitioner::InitClusters() { type = Cluster::KNOWN_SHAPE; } auto cluster = MakeShared(rank++, type, node, this); - REQUIRE_NOT_NULL(cluster, "Failed new memory for cluster."); + REQUIRE_NOT_NULL(cluster, "[New][Memory] for cluster failed."); node_2_cluster_[node] = cluster; - if (cluster->IsUnknownShape() && !cluster->IsControlFlow()) { + if (cluster->IsUnknownShape()) { ordered_cluster_.push_back(cluster); } + + int64_t group_index = -1; + if (AttrUtils::GetInt(node->GetOpDesc(), ATTR_NAME_CONTROL_FLOW_GROUP, group_index)) { + GELOGD("[%s] is rts control flow Op, group index: %ld", node->GetName().c_str(), group_index); + auto &control_cluster = control_clusters_[group_index]; + control_cluster.emplace_back(cluster); + } + // Already sorted topologically, so access to the parent cluster is safe for (const auto &parent : node->GetInAllNodes()) { cluster->AddInput(node_2_cluster_[parent]); @@ -350,14 +363,41 @@ static std::string ToString(const std::vector &clusters) { } } +void DynamicShapePartitioner::MergeClustersControlFlow() { + for (const auto &item : control_clusters_) { + const auto &control_cluster = item.second; + auto rit = control_cluster.rbegin(); + if (rit == control_cluster.rend()) { + GELOGW("Invalid empty control flow cluster."); + continue; + } + + const auto &cluster = *rit; + for (++rit; rit != control_cluster.rend(); ++rit) { + const auto &cluster_from = *rit; + auto merged_clusters = cluster->MergeAllPathFrom(cluster_from); + GELOGD("Merge all path cluster from %lu to %lu %s.", cluster_from->Id(), cluster->Id(), + ToString(merged_clusters).c_str()); + for (const auto &merged_cluster : merged_clusters) { + for (const auto &node : merged_cluster->Nodes()) { + node_2_cluster_[node] = cluster; + } + } + } + } +} + void DynamicShapePartitioner::MergeClustersUnknownShape() { // Merge unknown shape clusters for (const auto &cluster : ordered_cluster_) { - if (cluster->IsIndependent() || cluster->IsControlFlow()) { + if (cluster->IsIndependent()) { continue; } for (const auto &in_cluster : cluster->Inputs()) { - if (!in_cluster->IsUnknownShape() || in_cluster->IsControlFlow()) { + if (!in_cluster->IsUnknownShape()) { + continue; + } + if (!cluster->IsAdjoinNodes(in_cluster)) { continue; } auto merged_clusters = cluster->MergeAllPathFrom(in_cluster); @@ -419,8 +459,9 @@ void DynamicShapePartitioner::MergeClustersInputData() { } Status DynamicShapePartitioner::MergeClusters() { + MergeClustersControlFlow(); MergeClustersUnknownShape(); - REQUIRE_SUCCESS(TopologicalSortClusters(), "Failed topological sort clusters after merge unknown shape clusters."); + REQUIRE_SUCCESS(TopologicalSortClusters(), "[TopologicalSort][Clusters] after merge unknown shape clusters failed."); MergeClustersKnownShape(); MergeClustersInputData(); return SUCCESS; @@ -446,7 +487,7 @@ Status DynamicShapePartitioner::CollectSpreadUnknownShapeNodes(NodePtr node) { return SUCCESS; } auto opdesc = node->GetOpDesc(); - REQUIRE_NOT_NULL(opdesc, "Opdesc is nullptr."); + REQUIRE_NOT_NULL(opdesc, "[Get][OpDesc] Opdesc is nullptr."); // One can set 'ATTR_NAME_IS_UNKNOWN_SHAPE=true' on node so as to forcing the node flow into the unknown subgraph, // ignore the actual shape. if (JudgeUnknowShapeWithAttr(opdesc)) { @@ -492,10 +533,11 @@ Status DynamicShapePartitioner::CollectSpreadUnknownShapeNodes(NodePtr node) { auto graph = root_graph_; for (const auto &subgraph_name : opdesc->GetSubgraphInstanceNames()) { auto subgraph = graph->GetSubgraph(subgraph_name); - REQUIRE_NOT_NULL(subgraph, "Failed get subgraph %s of node %s on root graph.", subgraph_name.c_str(), + REQUIRE_NOT_NULL(subgraph, "[Get][Subgraph] %s of node %s on root graph failed.", subgraph_name.c_str(), node->GetName().c_str()); bool is_graph_unknow = false; - REQUIRE_SUCCESS(IsUnknownShapeGraph(subgraph, is_graph_unknow), "Failed check subgraph %s shape of node %s.", + REQUIRE_SUCCESS(IsUnknownShapeGraph(subgraph, is_graph_unknow), + "[Call][IsUnknownShapeGraph] Failed check subgraph %s shape of node %s.", subgraph_name.c_str(), node->GetName().c_str()); if (is_graph_unknow) { GELOGD("Collect node %s as its subgraph %s is unknown.", node->GetName().c_str(), subgraph->GetName().c_str()); @@ -526,9 +568,10 @@ Status DynamicShapePartitioner::IsUnknownShapeNode(NodePtr node, bool &is_unknow } for (auto &subgraph_name : opdesc->GetSubgraphInstanceNames()) { auto subgraph = graph->GetSubgraph(subgraph_name); - REQUIRE_NOT_NULL(subgraph, "Failed get subgraph %s of node %s on root graph.", subgraph_name.c_str(), + REQUIRE_NOT_NULL(subgraph, "[Get][Subgraph] %s of node %s on root graph failed.", subgraph_name.c_str(), node->GetName().c_str()); - REQUIRE_SUCCESS(IsUnknownShapeGraph(subgraph, is_unknown), "Failed check subgraph %s shape of node %s.", + REQUIRE_SUCCESS(IsUnknownShapeGraph(subgraph, is_unknown), + "[Call][IsUnknownShapeGraph] Failed check subgraph %s shape of node %s.", subgraph_name.c_str(), node->GetName().c_str()); if (is_unknown) { GELOGD("Mark node %s unknown as unknown subgraph.", node->GetName().c_str()); @@ -541,7 +584,8 @@ Status DynamicShapePartitioner::IsUnknownShapeNode(NodePtr node, bool &is_unknow Status DynamicShapePartitioner::IsUnknownShapeGraph(ComputeGraphPtr graph, bool &is_unknown) { for (auto &node : graph->GetDirectNode()) { - REQUIRE_SUCCESS(IsUnknownShapeNode(node, is_unknown), "Failed check node %s shape on graph %s.", + REQUIRE_SUCCESS(IsUnknownShapeNode(node, is_unknown), + "[Call][IsUnknownShapeNode]Failed check node %s shape on graph %s.", node->GetName().c_str(), graph->GetName().c_str()); if (is_unknown) { GELOGD("Mark graph %s unknown as contains unknown node %s.", graph->GetName().c_str(), node->GetName().c_str()); @@ -608,13 +652,6 @@ bool Cluster::IsRefVariable() const { return false; } -bool Cluster::IsControlFlow() const { - const auto &op_desc = nodes_[0]->GetOpDesc(); - bool is_ctrl_flow = kControlFlowOps.count(op_desc->GetType()) > 0 && op_desc->HasAttr(ATTR_NAME_FORCE_UNKNOWN_SHAPE); - GELOGD("[%s] %s rts control flow Op ", op_desc->GetName().c_str(), is_ctrl_flow ? "Is" : "Not"); - return is_ctrl_flow; -} - void Cluster::AddInput(ClusterPtr in) { if (std::find(in_clusters_.begin(), in_clusters_.end(), in) != in_clusters_.end()) return; in_clusters_.insert(in_clusters_.end(), in); @@ -694,10 +731,7 @@ std::vector Cluster::MergeAllPathFrom(ClusterPtr other) { if (other->IsIndependent()) { return path_clusters; } - if (std::find(other->out_clusters_.begin(), other->out_clusters_.end(), shared_from_this()) == - other->out_clusters_.end()) { - return path_clusters; - } + path_clusters.push_back(other); forward_reached_queue.push(other); backward_reached_queue.push(shared_from_this()); @@ -761,7 +795,7 @@ InControlAnchorPtr Cluster::GetFrameInControlAnchor() { return partition_node_-> OutControlAnchorPtr Cluster::GetFrameOutControlAnchor() { return partition_node_->GetOutControlAnchor(); }; Status Cluster::BuildFrame() { - if ((IsUnknownShape() || IsKnownShape() || IsInputNode()) && !IsControlFlow()) { + if (IsUnknownShape() || IsKnownShape() || IsInputNode()) { return BuildPartitionFrame(); } else { auto node = nodes_.front(); @@ -772,7 +806,7 @@ Status Cluster::BuildFrame() { if (src_cluster->id_ != id_) { REQUIRE_GRAPH_SUCCESS( GraphUtils::RemoveEdge(peer_out_control_anchor, in_control_anchor), - "Failed remove edge from node %s index %d to node %s index %d.", + "[Remove][Edge] from node %s index %d to node %s failed, index %d.", peer_out_control_anchor->GetOwnerNode()->GetName().c_str(), AnchorUtils::GetIdx(peer_out_control_anchor), in_control_anchor->GetOwnerNode()->GetName().c_str(), AnchorUtils::GetIdx(in_control_anchor)); control_inputs_.insert(src_cluster); @@ -802,20 +836,28 @@ Status Cluster::BuildPartitionFrame() { string sub_graph_name_patten = (is_input ? "_input" : known_name); std::string sub_graph_name = graph->GetName() + "_sub_" + std::to_string(unique_id_) + sub_graph_name_patten; subgraph_ = MakeShared(sub_graph_name); - REQUIRE_NOT_NULL(subgraph_, "Failed new memory for subgraph."); + REQUIRE_NOT_NULL(subgraph_, "[New][Memory] for subgraph failed, name:%s.", sub_graph_name.c_str()); auto partition_op = MakeShared("PartitionedCall_" + std::to_string(unique_id_++), "PartitionedCall"); - REQUIRE_NOT_NULL(partition_op, "Failed new memory for partition op."); + REQUIRE_NOT_NULL(partition_op, "[New][Memory] for partition op failed."); REQUIRE(AttrUtils::SetBool(partition_op, ATTR_NAME_IS_UNKNOWN_SHAPE, is_unknown_shape), - "Failed set _is_unknown_shape flag on partitioned op %s.", partition_op->GetName().c_str()); - REQUIRE_GRAPH_SUCCESS(partition_op->AddSubgraphName(subgraph_->GetName()), "Failed add subgraph name."); + "[Set][Attr] _is_unknown_shape flag on partitioned op %s failed.", partition_op->GetName().c_str()); + REQUIRE_GRAPH_SUCCESS(partition_op->AddSubgraphName(subgraph_->GetName()), + "[Add][SubgraphName] %s for op:%s.", + subgraph_->GetName().c_str(), partition_op->GetName().c_str()); REQUIRE_GRAPH_SUCCESS(partition_op->SetSubgraphInstanceName(0, subgraph_->GetName()), - "Failed set subgraph instance name."); + "[Call][SetSubgraphInstanceName] for op:%s failed, index:0, name:%s.", + partition_op->GetName().c_str(), subgraph_->GetName().c_str()); for (auto &node : nodes_) { - REQUIRE_NOT_NULL(subgraph_->AddNode(node), "Failed add node to subgraph."); + REQUIRE_NOT_NULL(subgraph_->AddNode(node), + "[Add][Node] %s to subgraph:%s failed.", node->GetName().c_str(), subgraph_->GetName().c_str()); REQUIRE(AttrUtils::SetBool(node->GetOpDesc(), ATTR_NAME_IS_UNKNOWN_SHAPE, is_unknown_shape), - "Failed set shape flag."); - REQUIRE_GRAPH_SUCCESS(GraphUtils::RemoveJustNode(graph, node), "Failed remove root graph node."); - REQUIRE_GRAPH_SUCCESS(node->SetOwnerComputeGraph(subgraph_), "Failed set owner graph."); + "[Set][Attr] %s to op:%s failed.", ATTR_NAME_IS_UNKNOWN_SHAPE.c_str(), node->GetName().c_str()); + REQUIRE_GRAPH_SUCCESS(GraphUtils::RemoveJustNode(graph, node), + "[Remove][JustNode] failed, graph:%s, node:%s.", + graph->GetName().c_str(), node->GetName().c_str()); + REQUIRE_GRAPH_SUCCESS(node->SetOwnerComputeGraph(subgraph_), + "[Set][OwnerComputeGraph] %s for node:%s failed.", + subgraph_->GetName().c_str(), node->GetName().c_str()); for (const auto &anchor : node->GetAllInDataAnchors()) { auto peer_out_anchor = anchor->GetPeerOutAnchor(); if (peer_out_anchor == nullptr) { @@ -825,7 +867,7 @@ Status Cluster::BuildPartitionFrame() { if (src_cluster->id_ != id_) { AddFrameInput(anchor); REQUIRE_GRAPH_SUCCESS(partition_op->AddInputDesc(node->GetOpDesc()->GetInputDesc(anchor->GetIdx())), - "Failed add input desc."); + "[Add][InputDesc] to op:%s failed.", partition_op->GetName().c_str()); } } auto in_control_anchor = node->GetInControlAnchor(); @@ -838,7 +880,7 @@ Status Cluster::BuildPartitionFrame() { if (src_cluster->id_ != id_) { REQUIRE_GRAPH_SUCCESS( GraphUtils::RemoveEdge(peer_out_control_anchor, in_control_anchor), - "Failed remove edge from %s:%d to %s:%d.", peer_out_control_anchor->GetOwnerNode()->GetName().c_str(), + "[Remove][Edge] from %s:%d to %s:%d failed.", peer_out_control_anchor->GetOwnerNode()->GetName().c_str(), peer_out_control_anchor->GetIdx(), node->GetName().c_str(), in_control_anchor->GetIdx()); control_inputs_.insert(src_cluster); src_cluster->control_outputs_.insert(peer_out_control_anchor); @@ -852,23 +894,28 @@ Status Cluster::BuildPartitionFrame() { if (src_cluster->id_ != id_) { AddFrameOutput(anchor); REQUIRE_GRAPH_SUCCESS(partition_op->AddOutputDesc(node->GetOpDesc()->GetOutputDesc(anchor->GetIdx())), - "Failed add output desc."); + "[Add][OutputDesc] to op:%s failed.", partition_op->GetName().c_str()); break; } } } } partition_node_ = graph->AddNode(partition_op); - REQUIRE_NOT_NULL(partition_node_, "Failed add partition node."); - REQUIRE_GRAPH_SUCCESS(partition_node_->SetOwnerComputeGraph(graph), "Failed set owner graph."); + REQUIRE_NOT_NULL(partition_node_, + "[Add][Node] %s to graph:%s failed.", partition_op->GetName().c_str(), graph->GetName().c_str()); + REQUIRE_GRAPH_SUCCESS(partition_node_->SetOwnerComputeGraph(graph), + "[Set][OwnerComputeGraph] %s for node:%s failed.", + graph->GetName().c_str(), partition_op->GetName().c_str()); subgraph_->SetParentNode(partition_node_); subgraph_->SetParentGraph(graph); - REQUIRE_GRAPH_SUCCESS(graph->AddSubgraph(subgraph_), "Failed add subgraph to root graph."); + REQUIRE_GRAPH_SUCCESS(graph->AddSubgraph(subgraph_), + "[Add][Subgraph] %s to root graph:%s failed.", + subgraph_->GetName().c_str(), graph->GetName().c_str()); std::string session_graph_id; REQUIRE(AttrUtils::GetStr(*graph, ATTR_NAME_SESSION_GRAPH_ID, session_graph_id), - "Failed get ATTR_NAME_SESSION_GRAPH_ID on root graph."); + "[Get][Attr] %s on root graph:%s failed.", ATTR_NAME_SESSION_GRAPH_ID.c_str(), graph->GetName().c_str()); REQUIRE(AttrUtils::SetStr(*subgraph_, ATTR_NAME_SESSION_GRAPH_ID, session_graph_id), - "Failed set ATTR_NAME_SESSION_GRAPH_ID on subgraph."); + "[Set][Attr] %s on subgraph:%s failed.", ATTR_NAME_SESSION_GRAPH_ID.c_str(), subgraph_->GetName().c_str()); return SUCCESS; } @@ -878,17 +925,17 @@ Status Cluster::CombinePartitionFrame() { auto src_cluster = partitioner_->node_2_cluster_[peer_out_anchor->GetOwnerNode()]; auto src_anchor = src_cluster->GetFrameOutDataAnchor(peer_out_anchor); auto dst_anchor = GetFrameInDataAnchor(anchor); - REQUIRE_GRAPH_SUCCESS(GraphUtils::RemoveEdge(peer_out_anchor, anchor), "Failed remove edge from %s:%d to %s:%d.", + REQUIRE_GRAPH_SUCCESS(GraphUtils::RemoveEdge(peer_out_anchor, anchor), "[Remove][Edge] from %s:%d to %s:%d fail.", peer_out_anchor->GetOwnerNode()->GetName().c_str(), peer_out_anchor->GetIdx(), anchor->GetOwnerNode()->GetName().c_str(), anchor->GetIdx()); - REQUIRE_GRAPH_SUCCESS(GraphUtils::AddEdge(src_anchor, dst_anchor), "Failed add edge from %s:%d to %s:%d.", + REQUIRE_GRAPH_SUCCESS(GraphUtils::AddEdge(src_anchor, dst_anchor), "[Add][Edge] from %s:%d to %s:%d failed.", src_anchor->GetOwnerNode()->GetName().c_str(), src_anchor->GetIdx(), dst_anchor->GetOwnerNode()->GetName().c_str(), dst_anchor->GetIdx()); } for (const auto &src_cluster : control_inputs_) { auto src_anchor = src_cluster->GetFrameOutControlAnchor(); auto dst_anchor = GetFrameInControlAnchor(); - REQUIRE_GRAPH_SUCCESS(GraphUtils::AddEdge(src_anchor, dst_anchor), "Failed add edge from %s:%d to %s:%d.", + REQUIRE_GRAPH_SUCCESS(GraphUtils::AddEdge(src_anchor, dst_anchor), "[Add][Edge] from %s:%d to %s:%d failed.", src_anchor->GetOwnerNode()->GetName().c_str(), src_anchor->GetIdx(), dst_anchor->GetOwnerNode()->GetName().c_str(), dst_anchor->GetIdx()); } @@ -896,62 +943,76 @@ Status Cluster::CombinePartitionFrame() { } Status Cluster::BuildPartitionSubgraph() { - if (IsData() || IsNetOutput() || IsIndependent() || IsControlFlow()) { + if (IsData() || IsNetOutput() || IsIndependent()) { return SUCCESS; } int64_t parent_node_index = 0; for (auto anchor : inputs_) { auto data_op = MakeShared(subgraph_->GetName() + std::string("Data_") + std::to_string(parent_node_index), ge::DATA); - REQUIRE_NOT_NULL(data_op, "Failed new memory for data op."); + REQUIRE_NOT_NULL(data_op, "[New][Memory] for data op failed."); auto input_desc = anchor->GetOwnerNode()->GetOpDesc()->GetInputDesc(anchor->GetIdx()); - REQUIRE_GRAPH_SUCCESS(data_op->AddInputDesc(input_desc), "Failed add input desc."); - REQUIRE_GRAPH_SUCCESS(data_op->AddOutputDesc(input_desc), "Failed add output desc."); + REQUIRE_GRAPH_SUCCESS(data_op->AddInputDesc(input_desc), + "[Add][InputDesc] to op:%s failed.", data_op->GetName().c_str()); + REQUIRE_GRAPH_SUCCESS(data_op->AddOutputDesc(input_desc), + "[Add][OutputDesc] to op:%s failed.", data_op->GetName().c_str()); REQUIRE(AttrUtils::SetInt(data_op, ATTR_NAME_PARENT_NODE_INDEX, parent_node_index), - "Failed set parent_node_index on subgraph data node."); + "[Set][Attr] %s on subgraph data node:%s failed.", + ATTR_NAME_PARENT_NODE_INDEX.c_str(), data_op->GetName().c_str()); bool is_unknown_shape = IsUnknownShape(); REQUIRE(AttrUtils::SetBool(data_op, ATTR_NAME_IS_UNKNOWN_SHAPE, is_unknown_shape), - "Failed set _is_unknown_shape flag on data op %s.", data_op->GetName().c_str()); + "[Set][Attr] %s on data op %s failed.", ATTR_NAME_IS_UNKNOWN_SHAPE.c_str(), data_op->GetName().c_str()); auto data_node = subgraph_->AddNode(data_op); - REQUIRE_NOT_NULL(data_node, "Failed add data node to subgraph."); - REQUIRE_GRAPH_SUCCESS(data_node->SetOwnerComputeGraph(subgraph_), "Failed set owner graph of data node."); + REQUIRE_NOT_NULL(data_node, + "[Add][Node] %s to subgraph:%s failed.", data_op->GetName().c_str(), subgraph_->GetName().c_str()); + REQUIRE_GRAPH_SUCCESS(data_node->SetOwnerComputeGraph(subgraph_), + "[Set][OwnerGraph] %s of data node:%s failed.", + subgraph_->GetName().c_str(), data_op->GetName().c_str()); REQUIRE_GRAPH_SUCCESS(GraphUtils::AddEdge(data_node->GetOutDataAnchor(0), anchor), - "Faile add data input edge to %s:%d", anchor->GetOwnerNode()->GetName().c_str(), - anchor->GetIdx()); + "[Call][AddEdge] Failed add data input edge to %s:%d", + anchor->GetOwnerNode()->GetName().c_str(), anchor->GetIdx()); parent_node_index++; } if (outputs_.empty() && control_outputs_.empty()) { return SUCCESS; } auto net_output_op = MakeShared(subgraph_->GetName() + "_" + NODE_NAME_NET_OUTPUT, ge::NETOUTPUT); - REQUIRE_NOT_NULL(net_output_op, "Failed new memory for netoutput op."); + REQUIRE_NOT_NULL(net_output_op, "[New][Memory] for netoutput op failed."); bool is_unknown_shape = IsUnknownShape(); REQUIRE(AttrUtils::SetBool(net_output_op, ATTR_NAME_IS_UNKNOWN_SHAPE, is_unknown_shape), - "Failed set _is_unknown_shape flag on net_output_op %s.", net_output_op->GetName().c_str()); + "[Set][Attr] %s on op:%s failed.", ATTR_NAME_IS_UNKNOWN_SHAPE.c_str(), net_output_op->GetName().c_str()); for (size_t i = 0; i < outputs_.size(); ++i) { GeTensorDesc input_desc; - REQUIRE_GRAPH_SUCCESS(net_output_op->AddInputDesc(input_desc), "Failed add input desc."); + REQUIRE_GRAPH_SUCCESS(net_output_op->AddInputDesc(input_desc), + "[Add][InputDesc] to op:%s failed.", net_output_op->GetName().c_str()); } auto net_output_node = subgraph_->AddNode(net_output_op); - REQUIRE_NOT_NULL(net_output_node, "Failed add netoutput node to subgraph."); - REQUIRE_GRAPH_SUCCESS(net_output_node->SetOwnerComputeGraph(subgraph_), "Failed set owner graph of netoutput node."); + REQUIRE_NOT_NULL(net_output_node, + "[Call][AddNode] Failed add netoutput node:%s to subgraph:%s.", + net_output_op->GetName().c_str(), subgraph_->GetName().c_str()); + REQUIRE_GRAPH_SUCCESS(net_output_node->SetOwnerComputeGraph(subgraph_), + "[Set][OwnerGraph] %s of netoutput node:%s failed.", + subgraph_->GetName().c_str(), net_output_node->GetName().c_str()); parent_node_index = 0; for (const auto &anchor : outputs_) { auto output_desc = anchor->GetOwnerNode()->GetOpDesc()->GetOutputDesc(static_cast(anchor->GetIdx())); REQUIRE(AttrUtils::SetInt(output_desc, ATTR_NAME_PARENT_NODE_INDEX, parent_node_index), - "Failed set parent_node_index on subgraph netoutput's input."); + "[Set][Attr] parent_node_index on subgraph node:%s netoutput's input failed.", + anchor->GetOwnerNode()->GetName().c_str()); REQUIRE_GRAPH_SUCCESS(net_output_op->UpdateInputDesc(parent_node_index, output_desc), - "Failed update input desc of netoutput node."); + "[Update][InputDesc] of netoutput node:%s failed.", net_output_op->GetName().c_str()); REQUIRE_GRAPH_SUCCESS(GraphUtils::AddEdge(anchor, net_output_node->GetInDataAnchor(parent_node_index)), - "Faile add edge from %s:%d to netoutput node.", anchor->GetOwnerNode()->GetName().c_str(), - anchor->GetIdx()); + "[Add][Edge] from %s:%d to netoutput node:%s failed.", + anchor->GetOwnerNode()->GetName().c_str(), anchor->GetIdx(), + net_output_op->GetName().c_str()); parent_node_index++; } for (const auto &anchor : control_outputs_) { REQUIRE_GRAPH_SUCCESS(GraphUtils::AddEdge(anchor, net_output_node->GetInControlAnchor()), - "Faile add control edge from %s:%d to netoutput node.", - anchor->GetOwnerNode()->GetName().c_str(), anchor->GetIdx()); + "[Add][ControlEdge] from %s:%d to netoutput node:%s failed.", + anchor->GetOwnerNode()->GetName().c_str(), anchor->GetIdx(), + net_output_op->GetName().c_str()); } return SUCCESS; } diff --git a/ge/graph/partition/dynamic_shape_partition.h b/ge/graph/partition/dynamic_shape_partition.h index 93f86d82..a17c4e4b 100644 --- a/ge/graph/partition/dynamic_shape_partition.h +++ b/ge/graph/partition/dynamic_shape_partition.h @@ -47,7 +47,6 @@ class DynamicShapePartitioner { bool IsUnknownShape() const; bool IsIndependent() const; bool IsNetOutput() const; - bool IsControlFlow() const; std::vector> Inputs() const; std::vector> Outputs() const; bool IsInputNode() const; @@ -81,6 +80,10 @@ class DynamicShapePartitioner { Status BuildPartitionSubgraph(); // Clear resource and break circular dependency void Clear(); + bool IsAdjoinNodes(const std::shared_ptr &other) const { + const auto &out_clusters = other->out_clusters_; + return std::find(out_clusters.begin(), out_clusters.end(), shared_from_this()) != out_clusters.end(); + } private: static thread_local size_t unique_id_; @@ -126,13 +129,15 @@ class DynamicShapePartitioner { // and there's only one path between the two clusters , merge the two clusters // 3) Iterate through the INPUT_DATA clusters, merge all INPUT_DATA Status MergeClusters(); + // Merge clusters step0 + void MergeClustersControlFlow(); // Merge clusters step1 void MergeClustersUnknownShape(); // Merge clusters step2 void MergeClustersKnownShape(); // Merge clusters step3 void MergeClustersInputData(); - // Topological sort clusters after merge unknow shape clusters. + // Topological sort clusters after merge unknown shape clusters. Status TopologicalSortClusters(); // Deduplicate merged clusters void PruneUniqueClusters(); @@ -140,7 +145,7 @@ class DynamicShapePartitioner { Status BuildPartitionFrame(); // Establish connection between corresponding partitioned of clusters Status CombinePartitionFrame(); - // Convert the nodes in cluster into a complete ComputeGraoh + // Convert the nodes in cluster into a complete ComputeGraph Status BuildPartitionSubgraph(); // Clear resource and break circular dependency void ClearResource(); @@ -155,6 +160,8 @@ class DynamicShapePartitioner { Status CtrlEdgeTransfer(); ge::ComputeGraphPtr root_graph_; // The original graph to partition std::unordered_map> node_2_cluster_; // Record nodes and the cluster it belongs to + // V1 control flow cluster, need merge to one Graph. + std::unordered_map>> control_clusters_; // topological sorted clusters, this field will change with the splitting. // When partitioning UNKNOWN_SHAPE cluster, it is a collection of all topological sorted UNKNOWN_SHAPE clusters // When partitioning KNOWN_SHAPE cluster, it is a collection of all topological sorted KNOWN_SHAPE clusters diff --git a/ge/graph/partition/engine_place.cc b/ge/graph/partition/engine_place.cc index f4ebbdca..93cc3e61 100755 --- a/ge/graph/partition/engine_place.cc +++ b/ge/graph/partition/engine_place.cc @@ -36,12 +36,14 @@ std::mutex check_support_cost_mutex; } Status EnginePlacer::Check() const { if (compute_graph_ == nullptr) { - GELOGE(GE_GRAPH_NULL_INPUT, "compute_graph_ is null."); + REPORT_INNER_ERROR("E19999", "compute_graph_ is nullptr, check invalid."); + GELOGE(GE_GRAPH_NULL_INPUT, "[Check][Param] compute_graph_ is nullptr."); return FAILED; } std::shared_ptr instance_ptr = ge::GELib::GetInstance(); if ((instance_ptr == nullptr) || (!instance_ptr->InitFlag())) { - GELOGE(GE_CLI_GE_NOT_INITIALIZED, "Run enginePlacer failed"); + REPORT_INNER_ERROR("E19999", "GELib instance is nullptr or it is not InitFlag, check invalid."); + GELOGE(GE_CLI_GE_NOT_INITIALIZED, "[Get][GELib] Run enginePlacer failed, because GELib is invalid."); return FAILED; } return SUCCESS; @@ -87,13 +89,13 @@ Status EnginePlacer::Run() { is_check_support_success = false; ErrorManager::GetInstance().ATCReportErrMessage( "E13003", {"opname", "optype"}, {op_desc->GetName(), op_desc->GetType()}); - GELOGE(GE_CLI_GE_NOT_INITIALIZED, "Can not find engine of op type %s", - node_ptr->GetOpDesc()->GetType().c_str()); + GELOGE(GE_CLI_GE_NOT_INITIALIZED, "[Check][Param] Can not find engine of op name %s type %s", + op_desc->GetName().c_str(), op_desc->GetType().c_str()); continue; } } if (AssignEngineAndLog(node_ptr, engine_name) != SUCCESS) { - GELOGE(GE_GRAPH_ASSIGN_ENGINE_FAILED, "[GraphPartitioner]: AssignEngineAndLog FAILED"); + GELOGE(GE_GRAPH_ASSIGN_ENGINE_FAILED, "[Call][AssignEngineAndLog] FAILED, node:%s", op_desc->GetName().c_str()); return FAILED; } } @@ -107,7 +109,8 @@ Status EnginePlacer::Run() { Status EnginePlacer::AssignEngineAndLog(ge::ConstNodePtr node_ptr, const std::string &engine_name) { if ((node_ptr == nullptr) || (node_ptr->GetOpDesc() == nullptr)) { - GELOGE(FAILED, "node_ptr is null."); + REPORT_INNER_ERROR("E19999", "Param node_ptr is nullptr or it's opdesc is nullptr, check invalid."); + GELOGE(FAILED, "[Check][Param] node_ptr is nullptr."); return FAILED; } diff --git a/ge/graph/partition/graph_partition.cc b/ge/graph/partition/graph_partition.cc index 1e6f6895..503979cb 100755 --- a/ge/graph/partition/graph_partition.cc +++ b/ge/graph/partition/graph_partition.cc @@ -49,13 +49,18 @@ Status ge::GraphPartitioner::CheckIfEnd2PldEmpty(ge::ComputeGraphPtr &output_mer if (graph_info_.partitions_.size() == kOneGraph) { auto partition = (*graph_info_.partitions_.begin()); if (partition.first == nullptr) { - GELOGE(GE_GRAPH_EMPTY_PARTITION, "[GraphPartitioner]: partition.first is null, engine name is %s", + REPORT_INNER_ERROR("E19999", "partition.first is nullptr, check invalid, engine name is %s", + partition.second.c_str()); + GELOGE(GE_GRAPH_EMPTY_PARTITION, "[Check][Param] partition.first is null, engine name is %s", partition.second.c_str()); return FAILED; } output_merged_compute_graph = partition.first; } else { // if placeholder to end map is empty, it should be an exception condition - GELOGE(GE_GRAPH_EMPTY_PARTITION, "[GraphPartitioner]: placeholder to end map is empty, partitions size is not 1."); + REPORT_INNER_ERROR("E19999", "partitions size:%zu is not 1, check invalid.", graph_info_.partitions_.size()); + GELOGE(GE_GRAPH_EMPTY_PARTITION, + "[Check][Param] placeholder to end map is empty, partitions size:%zu is not 1.", + graph_info_.partitions_.size()); return FAILED; } return SUCCESS; @@ -80,7 +85,8 @@ Status ge::GraphPartitioner::MergeAllSubGraph(ge::ComputeGraphPtr &output_merged (void)AttrUtils::SetStr(node->GetOpDesc(), ATTR_NAME_STREAM_LABEL, temp_stream); } if (node->SetOwnerComputeGraph(output_merged_compute_graph) != GRAPH_SUCCESS) { - GELOGE(GE_GRAPH_PARAM_NULLPTR, "SetownerComputeGraph failed, node %s", node->GetName().c_str()); + REPORT_CALL_ERROR("E19999", "SetOwnerComputeGraph for node:%s failed.", node->GetName().c_str()); + GELOGE(GE_GRAPH_PARAM_NULLPTR, "[Set][OwnerComputeGraph] failed, node %s", node->GetName().c_str()); return FAILED; } (void)output_merged_compute_graph->AddNode(node); @@ -111,7 +117,8 @@ Status ge::GraphPartitioner::RemoveNodeAndEdgeBetweenEndPld(ge::ComputeGraphPtr const std::vector &sub_graph_list) { if ((output_merged_compute_graph == nullptr) || (MergeAllSubGraph(output_merged_compute_graph, sub_graph_list) != SUCCESS)) { - GELOGE(GE_GRAPH_PARAM_NULLPTR, "[GraphPartitioner]: MergeAllSubGraph failed."); + REPORT_INNER_ERROR("E19999", "output_merged_compute_graph is nullptr or Call MergeAllSubGraph failed."); + GELOGE(GE_GRAPH_PARAM_NULLPTR, "[Merge][AllSubGraph] failed."); return FAILED; } for (const auto &it : graph_info_.index_2_end_) { @@ -127,19 +134,31 @@ Status ge::GraphPartitioner::RemoveNodeAndEdgeBetweenEndPld(ge::ComputeGraphPtr : Anchor::DynamicAnchorCast(pld->GetOutDataAnchor(0)); auto src_anchor = end_in_anchor->GetFirstPeerAnchor(); // src_anchor should be only 1 if (GraphUtils::RemoveEdge(src_anchor, end_in_anchor) != GRAPH_SUCCESS) { - GELOGE(GE_GRAPH_PARAM_NULLPTR, "[GraphPartitioner]: RemoveEdge failed. node_name:%s, graph_name:%s", + REPORT_CALL_ERROR("E19999", "RemoveEdge between %s and %s failed", + src_anchor->GetOwnerNode()->GetName().c_str(), + end_in_anchor->GetOwnerNode()->GetName().c_str()); + GELOGE(GE_GRAPH_PARAM_NULLPTR, "[Remove][Edge] between %s and %s failed. node_name:%s, graph_name:%s", + src_anchor->GetOwnerNode()->GetName().c_str(), end_in_anchor->GetOwnerNode()->GetName().c_str(), end->GetName().c_str(), end->GetOwnerComputeGraph()->GetName().c_str()); return FAILED; } GE_CHECK_NOTNULL(pld_out_anchor); for (const auto &peer_in_anchor : pld_out_anchor->GetPeerAnchors()) { if (GraphUtils::RemoveEdge(pld_out_anchor, peer_in_anchor) != GRAPH_SUCCESS) { - GELOGE(GE_GRAPH_PARAM_NULLPTR, "[GraphPartitioner]: RemoveEdge failed. node_name:%s, graph_name:%s", + REPORT_CALL_ERROR("E19999", "RemoveEdge between %s and %s failed", + pld_out_anchor->GetOwnerNode()->GetName().c_str(), + peer_in_anchor->GetOwnerNode()->GetName().c_str()); + GELOGE(GE_GRAPH_PARAM_NULLPTR, "[Remove][Edge] between %s and %s failed. node_name:%s, graph_name:%s", + pld_out_anchor->GetOwnerNode()->GetName().c_str(), peer_in_anchor->GetOwnerNode()->GetName().c_str(), pld->GetName().c_str(), pld->GetOwnerComputeGraph()->GetName().c_str()); return FAILED; } if (GraphUtils::AddEdge(src_anchor, peer_in_anchor) != GRAPH_SUCCESS) { - GELOGE(GE_GRAPH_PARAM_NULLPTR, "merge two subgraph fail."); + REPORT_CALL_ERROR("E19999", "AddEdge from %s to %s failed.", + src_anchor->GetOwnerNode()->GetName().c_str(), + peer_in_anchor->GetOwnerNode()->GetName().c_str()); + GELOGE(GE_GRAPH_PARAM_NULLPTR, "[Add][Edge] from %s to %s failed.", + src_anchor->GetOwnerNode()->GetName().c_str(), peer_in_anchor->GetOwnerNode()->GetName().c_str()); return FAILED; } } @@ -157,7 +176,7 @@ Status ge::GraphPartitioner::MergeAfterSubGraphOptimization(ge::ComputeGraphPtr if (ret != SUCCESS) { // even though failed, ensure all op do finish check support real_ret = FAILED; - GELOGE(ret, "Graph merging Failed"); + GELOGE(ret, "[Merge][SubGraph] Failed, ret:%d", ret); } GE_CHECK_NOTNULL(original_compute_graph); // partition sub graph @@ -166,7 +185,7 @@ Status ge::GraphPartitioner::MergeAfterSubGraphOptimization(ge::ComputeGraphPtr ret = MergeSubGraph(merged_sub_graph, sub_graph); if (ret != SUCCESS) { real_ret = FAILED; - GELOGE(ret, "Sub graph merging Failed"); + GELOGE(ret, "[Merge][SubGraph] Failed, ret:%d", ret); continue; } // add sub graph @@ -176,19 +195,32 @@ Status ge::GraphPartitioner::MergeAfterSubGraphOptimization(ge::ComputeGraphPtr merged_sub_graph->SetOutputSize(sub_graph->GetOutputSize()); auto parent_node = sub_graph->GetParentNode(); GE_IF_BOOL_EXEC(parent_node == nullptr, - GELOGE(FAILED, "Parent node is null, graph name is %s", sub_graph->GetName().c_str()); + REPORT_INNER_ERROR("E19999", "Parent node of graph:%s is nullptr.", + sub_graph->GetName().c_str()); + GELOGE(FAILED, "[Check][Param] Parent node is null, graph name is %s", + sub_graph->GetName().c_str()); return FAILED;) auto original_graph = parent_node->GetOwnerComputeGraph(); GE_IF_BOOL_EXEC(graph_2_graph_partition_info_.find(original_graph) == graph_2_graph_partition_info_.end(), - GELOGE(FAILED, "Find graph info failed, graph name is %s", original_graph->GetName().c_str()); + REPORT_INNER_ERROR("E19999", "graph:%s not find in graph_2_graph_partition_info_, check invalid.", + original_graph->GetName().c_str()); + GELOGE(FAILED, "[Check][Param] Find graph info failed, graph name is %s", + original_graph->GetName().c_str()); return FAILED;) auto graph_info = graph_2_graph_partition_info_[original_graph]; GE_IF_BOOL_EXEC(graph_info.corresponding_node_in_partitions_.count(parent_node) == 0, - GELOGE(FAILED, "Find corresponding node failed, parent node name is %s", parent_node->GetName().c_str()); - return FAILED;) + REPORT_INNER_ERROR("E19999", "node:%s not find in corresponding_node_in_partitions_, " + "check invalid", parent_node->GetName().c_str()); + GELOGE(FAILED, "[Check][Param] Find corresponding node failed, parent node name is %s", + parent_node->GetName().c_str()); + return FAILED;) auto corresponding_node = graph_info.corresponding_node_in_partitions_[parent_node]; - GE_IF_BOOL_EXEC(corresponding_node == nullptr, GELOGE(FAILED, "Get null node, node name is %s", - parent_node->GetName().c_str()); return FAILED;); + GE_IF_BOOL_EXEC(corresponding_node == nullptr, + REPORT_INNER_ERROR("E19999", "Get null node in corresponding_node_in_partitions_, " + "first node name is %s", parent_node->GetName().c_str()); + GELOGE(FAILED, "[Check][Param] Get null node in corresponding_node_in_partitions_, " + "first node name is %s", parent_node->GetName().c_str()); + return FAILED;); merged_sub_graph->SetParentNode(corresponding_node); auto subgraph_parent_graph = corresponding_node->GetOwnerComputeGraph(); merged_sub_graph->SetParentGraph(subgraph_parent_graph); @@ -207,12 +239,17 @@ Status ge::GraphPartitioner::MergeAfterSubGraphOptimization(ge::ComputeGraphPtr Status ge::GraphPartitioner::MergeSubGraph(ge::ComputeGraphPtr &output_merged_compute_graph, const ge::ComputeGraphPtr &original_compute_graph) { if (original_compute_graph == nullptr) { - GELOGE(GE_GRAPH_NULL_INPUT, "[GraphPartitioner]: compute_graph is null."); + REPORT_INNER_ERROR("E19999", "Param original_compute_graph is nullptr, check invalid."); + GELOGE(GE_GRAPH_NULL_INPUT, "[Check][Param] original_compute_graph is nullptr."); return FAILED; } if ((graph_2_graph_partition_info_.find(original_compute_graph) == graph_2_graph_partition_info_.end()) || (graph_2_subgraph_list_.find(original_compute_graph) == graph_2_subgraph_list_.end())) { - GELOGE(GE_GRAPH_NULL_INPUT, "[GraphPartitioner]: compute_graph is error."); + REPORT_INNER_ERROR("E19999", "original_compute_graph:%s is not find in graph_2_graph_partition_info_.", + original_compute_graph->GetName().c_str()); + GELOGE(GE_GRAPH_NULL_INPUT, + "[Check][Param] original_compute_graph:%s is not find in graph_2_graph_partition_info_.", + original_compute_graph->GetName().c_str()); return FAILED; } GraphPartitionInfo &subgraph_info = graph_2_graph_partition_info_[original_compute_graph]; @@ -220,14 +257,16 @@ Status ge::GraphPartitioner::MergeSubGraph(ge::ComputeGraphPtr &output_merged_co graph_info_ = subgraph_info; if (graph_info_.mode_ != kMerging) { - GELOGE(GE_GRAPH_UNSUPPORTED, "Cannot call merging in partition mode"); + REPORT_INNER_ERROR("E19999", "Cannot call merging in partition mode, as mode != %d", kMerging); + GELOGE(GE_GRAPH_UNSUPPORTED, "[Check][Param] Cannot call merging in partition mode, as mode != %d", kMerging); return FAILED; } GELOGD("Graph merge starts."); // check input param for (const auto &it : sub_graph_list) { if (it == nullptr) { - GELOGE(GE_GRAPH_PARAM_NULLPTR, "[GraphPartitioner]: merging sub-graphs failed, sub-graph is null"); + REPORT_INNER_ERROR("E19999", "sub_graph is nullptr, check invalid."); + GELOGE(GE_GRAPH_PARAM_NULLPTR, "[Check][Param] merging sub-graphs failed, sub-graph is nullptr"); return FAILED; } } @@ -242,14 +281,16 @@ Status ge::GraphPartitioner::MergeSubGraph(ge::ComputeGraphPtr &output_merged_co output_merged_compute_graph = new_sub_graph; GE_TIMESTAMP_START(MergeSubGraphRemoveNode); if (RemoveNodeAndEdgeBetweenEndPld(output_merged_compute_graph, sub_graph_list) != ge::SUCCESS) { - GELOGE(GE_GRAPH_PARAM_NULLPTR, "[GraphPartitioner]: merging sub-graphs failed"); + GELOGE(GE_GRAPH_PARAM_NULLPTR, "[Call][RemoveNodeAndEdgeBetweenEndPld] failed, graph:%s", + output_merged_compute_graph->GetName().c_str()); return FAILED; } GE_TIMESTAMP_END(MergeSubGraphRemoveNode, "GraphPartitioner::MergeGraphRemoveNodeAndEdge"); GE_TIMESTAMP_START(MergeSubGraphTopologicalSorting); Status ret = output_merged_compute_graph->TopologicalSorting(); if (ret != SUCCESS) { - GELOGE(GE_GRAPH_TOPO_SORT_FAILED, "[GraphPartitioner]: output_merged_compute_graph->TopologicalSorting failed"); + GELOGE(GE_GRAPH_TOPO_SORT_FAILED, "[Call][TopologicalSorting] for output_merged_compute_graph:%s failed", + output_merged_compute_graph->GetName().c_str()); return FAILED; } GE_TIMESTAMP_END(MergeSubGraphTopologicalSorting, "GraphPartitioner::MergeGraphTopologicalSorting"); @@ -257,7 +298,8 @@ Status ge::GraphPartitioner::MergeSubGraph(ge::ComputeGraphPtr &output_merged_co GE_TIMESTAMP_START(MergeSubGraphEnginePlacerRun); graph_info_.engine_placer_.SetComputeGraph(output_merged_compute_graph); if (graph_info_.engine_placer_.Run() != SUCCESS) { - GELOGE(GE_GRAPH_INIT_FAILED, "[GraphPartitioner]: engine_placer run failed"); + GELOGE(GE_GRAPH_INIT_FAILED, "[Call][Run] engine_placer run failed, graph:%s", + output_merged_compute_graph->GetName().c_str()); return FAILED; } GE_TIMESTAMP_END(MergeSubGraphEnginePlacerRun, "GraphPartitioner::MergeGraphEnginePlacerRun"); @@ -267,17 +309,22 @@ Status ge::GraphPartitioner::MergeSubGraph(ge::ComputeGraphPtr &output_merged_co Status ge::GraphPartitioner::UpdatePldOpDesc(const NodePtr &dst_node, int input_index, OpDescPtr &pld_op_desc) { if ((dst_node == nullptr) || (pld_op_desc == nullptr) || (dst_node->GetOpDesc() == nullptr)) { - GELOGE(FAILED, "parameter ptr is null."); + REPORT_INNER_ERROR("E19999", "Param dst_node or pld_op_desc or op of dst_node is nullptr, check invalid"); + GELOGE(FAILED, "[Check][Param] parameter ptr is null."); return FAILED; } const auto &input_desc = dst_node->GetOpDesc()->GetInputDesc(static_cast(input_index)); - GE_IF_BOOL_EXEC(pld_op_desc->AddOutputDesc(input_desc) != GRAPH_SUCCESS, GELOGE(FAILED, "AddOutputDesc failed"); - return FAILED;) + GE_IF_BOOL_EXEC(pld_op_desc->AddOutputDesc(input_desc) != GRAPH_SUCCESS, + REPORT_CALL_ERROR("E19999", "AddOutputDesc to op:%s failed", pld_op_desc->GetName().c_str()); + GELOGE(FAILED, "[Add][OutputDesc] to op:%s failed", pld_op_desc->GetName().c_str()); + return FAILED;) if (pld_op_desc->MutableOutputDesc(0) != nullptr) { ge::TensorUtils::SetRealDimCnt(*(pld_op_desc->MutableOutputDesc(0).get()), static_cast(input_desc.GetShape().GetDims().size())); } else { - GELOGE(GE_GRAPH_ADD_PLC_END_FAILED, "[GraphPartitioner]: pld_op_desc is null."); + REPORT_INNER_ERROR("E19999", "output(0) of op:%s is nullptr, check invalid", pld_op_desc->GetName().c_str()); + GELOGE(GE_GRAPH_ADD_PLC_END_FAILED, "[Check][Param] output(0) of op:%s is nullptr.", + pld_op_desc->GetName().c_str()); return FAILED; } return SUCCESS; @@ -285,17 +332,22 @@ Status ge::GraphPartitioner::UpdatePldOpDesc(const NodePtr &dst_node, int input_ Status ge::GraphPartitioner::UpdateEndOpDesc(const NodePtr &src_node, int output_index, OpDescPtr &end_op_desc) { if ((src_node == nullptr) || (end_op_desc == nullptr) || (src_node->GetOpDesc() == nullptr)) { - GELOGE(FAILED, "parameter ptr is null."); + REPORT_INNER_ERROR("E19999", "Param src_node or end_op_desc or op of src_node is nullptr, check invalid."); + GELOGE(FAILED, "[Check][Param] parameter ptr is null."); return FAILED; } const auto &output_desc = src_node->GetOpDesc()->GetOutputDesc(static_cast(output_index)); - GE_IF_BOOL_EXEC(end_op_desc->AddInputDesc(output_desc) != GRAPH_SUCCESS, GELOGE(FAILED, "AddInputDesc failed"); - return FAILED;) + GE_IF_BOOL_EXEC(end_op_desc->AddInputDesc(output_desc) != GRAPH_SUCCESS, + REPORT_CALL_ERROR("E19999", "AddInputDesc to op:%s failed", end_op_desc->GetName().c_str()); + GELOGE(FAILED, "[Add][InputDesc] to op:%s failed", end_op_desc->GetName().c_str()); + return FAILED;) if (end_op_desc->MutableInputDesc(0) != nullptr) { ge::TensorUtils::SetRealDimCnt(*(end_op_desc->MutableInputDesc(0).get()), static_cast(output_desc.GetShape().GetDims().size())); } else { - GELOGE(GE_GRAPH_ADD_PLC_END_FAILED, "[GraphPartitioner]: pld_op_desc is null."); + REPORT_INNER_ERROR("E19999", "input(0) of op:%s is nullptr, check invalid.", end_op_desc->GetName().c_str()); + GELOGE(GE_GRAPH_ADD_PLC_END_FAILED, "[Check][Param] input(0) of op:%s is nullptr.", + end_op_desc->GetName().c_str()); return FAILED; } return SUCCESS; @@ -315,7 +367,8 @@ graphStatus ge::GraphPartitioner::AddPlaceHolderEndInSrcDstGraph(const AnchorPtr string end_name = kEndType + std::to_string(graph_info_.num_of_pld_end_); auto end_op_desc = MakeShared(end_graph->GetName() + "_" + end_name, END); if (end_op_desc == nullptr) { - GELOGE(GRAPH_PARAM_INVALID, "pld_op_desc is nullptr."); + REPORT_CALL_ERROR("E19999", "New Memory for OpDesc failed."); + GELOGE(GRAPH_PARAM_INVALID, "[New][Memory] for OpDesc failed, pld_op_desc is nullptr."); return FAILED; } GE_IF_BOOL_EXEC(!AttrUtils::SetInt(end_op_desc, "peerIndex", graph_info_.num_of_pld_end_), @@ -333,28 +386,40 @@ graphStatus ge::GraphPartitioner::AddPlaceHolderEndInSrcDstGraph(const AnchorPtr bool is_need_update_desc = (output_index >= 0) && (graph_info_.mode_ == kPartitioning); if (is_need_update_desc) { if (UpdateEndOpDesc(src_node, output_index, end_op_desc) != SUCCESS) { - GELOGE(GRAPH_PARAM_INVALID, "UpdateEndOpDesc failed, input index %d", output_index); + GELOGE(GRAPH_PARAM_INVALID, "[Update][EndOpDesc] failed, input index:%d, end_op_desc:%s", + output_index, end_op_desc->GetName().c_str()); return FAILED; } } else { GeTensorDesc input_desc; if (end_op_desc->AddInputDesc(input_desc) != SUCCESS) { - GELOGE(GRAPH_PARAM_INVALID, "AddInputDesc failed, input index %d", output_index); + REPORT_CALL_ERROR("E19999", "add input desc to op:%s failed, input index:%d", + end_op_desc->GetName().c_str(), output_index); + GELOGE(GRAPH_PARAM_INVALID, "[Add][InputDesc] to op:%s failed, input index %d", + end_op_desc->GetName().c_str(), output_index); return FAILED; } } NodePtr new_end_node = end_graph->AddNode(end_op_desc); if (new_end_node == nullptr) { - GELOGE(GRAPH_PARAM_INVALID, "new_end_node is nullptr."); + REPORT_CALL_ERROR("E19999", "add node:%s in graph:%s failed", + end_op_desc->GetName().c_str(), end_graph->GetName().c_str()); + GELOGE(GRAPH_PARAM_INVALID, "[Add][Node] %s in graph:%s failed.", + end_op_desc->GetName().c_str(), end_graph->GetName().c_str()); return FAILED; } GE_IF_BOOL_EXEC(new_end_node->SetOwnerComputeGraph(end_graph) != GRAPH_SUCCESS, - GELOGE(GRAPH_PARAM_INVALID, "SetOwnerComputeGraph failed"); + REPORT_CALL_ERROR("E19999", "SetOwnerComputeGraph %s for node:%s failed", + end_graph->GetName().c_str(), new_end_node->GetName().c_str()); + GELOGE(GRAPH_PARAM_INVALID, "[Set][OwnerComputeGraph] %s for node:%s failed", + end_graph->GetName().c_str(), new_end_node->GetName().c_str()); return FAILED;) AnchorPtr end_dst_anchor = GetEndInAnchor(out_anchor, new_end_node); if (GraphUtils::AddEdge(out_anchor, end_dst_anchor) != GRAPH_SUCCESS) { - GELOGE(GE_GRAPH_ADD_PLC_END_FAILED, "add end node : %s node %dth out-anchor --> end in %s subgraph fail.", - src_node->GetName().c_str(), AnchorUtils::GetIdx(out_anchor), end_graph->GetName().c_str()); + REPORT_CALL_ERROR("E19999", "add edge from %s to %s failed", out_anchor->GetOwnerNode()->GetName().c_str(), + end_dst_anchor->GetOwnerNode()->GetName().c_str()); + GELOGE(GE_GRAPH_ADD_PLC_END_FAILED, "[Add][Edge] from %s to %s failed", + out_anchor->GetOwnerNode()->GetName().c_str(), end_dst_anchor->GetOwnerNode()->GetName().c_str()); return FAILED; } /// For fe, op id has been set in AddNode, @@ -365,7 +430,8 @@ graphStatus ge::GraphPartitioner::AddPlaceHolderEndInSrcDstGraph(const AnchorPtr const string pld_name = kPlaceHolderType + std::to_string(graph_info_.num_of_pld_end_); auto pld_op_desc = MakeShared(pld_graph->GetName() + "_" + pld_name, PLACEHOLDER); if (pld_op_desc == nullptr) { - GELOGE(GRAPH_PARAM_INVALID, "pld_op_desc is nullptr."); + REPORT_CALL_ERROR("E19999", "New Memory for OpDesc failed."); + GELOGE(GRAPH_PARAM_INVALID, "[New][Memory] for OpDesc failed."); return FAILED; } GE_IF_BOOL_EXEC(!AttrUtils::SetInt(pld_op_desc, "peerIndex", graph_info_.num_of_pld_end_), @@ -401,30 +467,42 @@ graphStatus ge::GraphPartitioner::AddPlaceHolderEndInSrcDstGraph(const AnchorPtr is_need_update_desc = (input_index >= 0) && (graph_info_.mode_ == kPartitioning); if (is_need_update_desc) { if (UpdatePldOpDesc(dst_node, input_index, pld_op_desc) != SUCCESS) { - GELOGE(GRAPH_PARAM_INVALID, "UpdateEndOpDesc failed, output index %d", input_index); + GELOGE(GRAPH_PARAM_INVALID, "[Update][PldOpDesc] failed, output index:%d, pld_op_desc:%s", + input_index, pld_op_desc->GetName().c_str()); return FAILED; } } else { GeTensorDesc output_desc; if (pld_op_desc->AddOutputDesc(output_desc) != SUCCESS) { - GELOGE(GRAPH_PARAM_INVALID, "AddOutputDesc failed, input index %d", input_index); + REPORT_CALL_ERROR("E19999", "AddOutputDesc to op:%s failed, input index %d", + pld_op_desc->GetName().c_str(), input_index); + GELOGE(GRAPH_PARAM_INVALID, "[Add][OutputDesc] to op:%s failed, input index %d", + pld_op_desc->GetName().c_str(), input_index); return FAILED; } } NodePtr new_pld_node = pld_graph->AddNode(pld_op_desc); if (new_pld_node == nullptr) { - GELOGE(GRAPH_PARAM_INVALID, "new_pld_node is nullptr."); + REPORT_CALL_ERROR("E19999", "AddNode %s in graph:%s failed.", + pld_op_desc->GetName().c_str(), pld_graph->GetName().c_str()); + GELOGE(GRAPH_PARAM_INVALID, "[Add][Node] %s in graph:%s failed.", + pld_op_desc->GetName().c_str(), pld_graph->GetName().c_str()); return FAILED; } GE_IF_BOOL_EXEC(new_pld_node->SetOwnerComputeGraph(pld_graph) != GRAPH_SUCCESS, - GELOGE(GRAPH_PARAM_INVALID, "SetOwnerComputeGraph failed"); + REPORT_CALL_ERROR("E19999", "SetOwnerComputeGraph for node:%s failed, graph:%s", + new_pld_node->GetName().c_str(), pld_graph->GetName().c_str()); + GELOGE(GRAPH_PARAM_INVALID, "[Set][OwnerComputeGraph] for node:%s failed, graph:%s", + new_pld_node->GetName().c_str(), pld_graph->GetName().c_str()); return FAILED;) AnchorPtr pld_src_anchor = GetPldOutAnchor(new_pld_node, peer_in_anchor); // link placeHolder -> computeNode if (GraphUtils::AddEdge(pld_src_anchor, peer_in_anchor) != GRAPH_SUCCESS) { - GELOGE(GE_GRAPH_ADD_PLC_END_FAILED, - "add placeholder node : placeholder --> %s node %dth in-anchor in %s subgraph fail.", - dst_node->GetName().c_str(), AnchorUtils::GetIdx(peer_in_anchor), pld_graph->GetName().c_str()); + REPORT_CALL_ERROR("E19999", "AddEdge from %s to %s failed", + pld_src_anchor->GetOwnerNode()->GetName().c_str(), + peer_in_anchor->GetOwnerNode()->GetName().c_str()); + GELOGE(GE_GRAPH_ADD_PLC_END_FAILED, "[Add][Edge] from %s to %s failed", + pld_src_anchor->GetOwnerNode()->GetName().c_str(), peer_in_anchor->GetOwnerNode()->GetName().c_str()); return FAILED; } graph_info_.index_2_end_[graph_info_.num_of_pld_end_] = new_end_node; @@ -436,7 +514,8 @@ graphStatus ge::GraphPartitioner::AddPlaceHolderEndInSrcDstGraph(const AnchorPtr Status ge::GraphPartitioner::LinkInput2EndRemoveOrginalLink(ge::NodePtr input_node, ge::ComputeGraphPtr src_graph, ge::ComputeGraphPtr dst_graph) { if ((input_node == nullptr) || (src_graph == nullptr) || (dst_graph == nullptr)) { - GELOGE(FAILED, "parameter ptr is null."); + REPORT_INNER_ERROR("E19999", "Param input_node or src_graph or dst_graph is nullptr, check invalid."); + GELOGE(FAILED, "[Check][Param] parameter input_node or src_graph or dst_graph is nullptr."); return FAILED; } // get the original anchors and remove the original link @@ -444,27 +523,40 @@ Status ge::GraphPartitioner::LinkInput2EndRemoveOrginalLink(ge::NodePtr input_no for (auto &peer_in_anchor : out_data_anchor->GetPeerAnchors()) { if (peer_in_anchor->GetOwnerNode()->GetType() != kEndType) { if (GraphUtils::RemoveEdge(out_data_anchor, peer_in_anchor) != GRAPH_SUCCESS) { - GELOGE(FAILED, "[GraphPartitioner]: RemoveEdge() failed."); + REPORT_CALL_ERROR("E19999", "RemoveEdge between %s and %s failed.", + out_data_anchor->GetOwnerNode()->GetName().c_str(), + peer_in_anchor->GetOwnerNode()->GetName().c_str()); + GELOGE(FAILED, "[Remove][Edge] between %s and %s failed.", + out_data_anchor->GetOwnerNode()->GetName().c_str(), peer_in_anchor->GetOwnerNode()->GetName().c_str()); return FAILED; } // link input -> end auto ret = AddPlaceHolderEndInSrcDstGraph(out_data_anchor, peer_in_anchor, src_graph, dst_graph); if (ret != SUCCESS) { - GELOGE(GE_GRAPH_ADD_PLC_END_FAILED, "[GraphPartitioner]: AddPlaceHolderEndInSrcDstGraph() failed."); + GELOGE(GE_GRAPH_ADD_PLC_END_FAILED, "[Call][AddPlaceHolderEndInSrcDstGraph] failed, ret:%d.", ret); return ret; } } else { auto end_node = peer_in_anchor->GetOwnerNode(); if (GraphUtils::RemoveJustNode(src_graph, end_node) != GRAPH_SUCCESS) { - GELOGE(FAILED, "[GraphPartitioner]: RemoveJustNode() failed."); + REPORT_CALL_ERROR("E19999", "RemoveJustNode %s from graph:%s failed.", + end_node->GetName().c_str(), src_graph->GetName().c_str()); + GELOGE(FAILED, "[Remove][JustNode] %s from graph:%s failed.", + end_node->GetName().c_str(), src_graph->GetName().c_str()); return FAILED; } if (end_node->SetOwnerComputeGraph(dst_graph) != GRAPH_SUCCESS) { - GELOGE(FAILED, "[GraphPartitioner]: RemoveJustNode() failed."); + REPORT_CALL_ERROR("E19999", "SetOwnerComputeGraph for node:%s failed, graph:%s.", + end_node->GetName().c_str(), dst_graph->GetName().c_str()); + GELOGE(FAILED, "[Set][OwnerComputeGraph] to node:%s failed, graph:%s.", + end_node->GetName().c_str(), dst_graph->GetName().c_str()); return FAILED; } if (dst_graph->AddNode(end_node) == nullptr) { - GELOGE(FAILED, "[GraphPartitioner]: AddNode() failed."); + REPORT_CALL_ERROR("E19999", "AddNode %s in graph:%s failed.", + end_node->GetName().c_str(), dst_graph->GetName().c_str()); + GELOGE(FAILED, "[Add][Node] %s in graph:%s failed.", + end_node->GetName().c_str(), dst_graph->GetName().c_str()); return FAILED; } } @@ -476,27 +568,37 @@ Status ge::GraphPartitioner::LinkInput2EndRemoveOrginalLink(ge::NodePtr input_no Status ge::GraphPartitioner::PutInputNodesInSubGraph(const ge::ComputeGraphPtr &src_graph, const ge::ComputeGraphPtr &dst_graph) { if ((src_graph == nullptr) || (dst_graph == nullptr)) { - GELOGE(FAILED, "parameter ptr is null."); + REPORT_INNER_ERROR("E19999", "Param src_graph or dst_graph is nullptr, check invalid."); + GELOGE(FAILED, "[Check][Param] parameter src_graph or dst_graph is nullptr."); return FAILED; } for (auto &input_node : src_graph->GetDirectNode()) { if (IsDataLike(input_node)) { if (input_node->SetOwnerComputeGraph(dst_graph) != GRAPH_SUCCESS) { - GELOGE(FAILED, "[GraphPartitioner]: SetOwnerComputeGraph failed."); + REPORT_CALL_ERROR("E19999", "SetOwnerComputeGraph for node:%s failed, graph:%s.", + input_node->GetName().c_str(), dst_graph->GetName().c_str()); + GELOGE(FAILED, "[Set][OwnerComputeGraph] for node:%s failed, graph:%s.", + input_node->GetName().c_str(), dst_graph->GetName().c_str()); return FAILED; } // remove input node from src_graph if (GraphUtils::RemoveJustNode(src_graph, input_node) != GRAPH_SUCCESS) { - GELOGE(FAILED, "[GraphPartitioner]: RemoveJustNode() failed."); + REPORT_CALL_ERROR("E19999", "RemoveJustNode %s from graph:%s failed.", + input_node->GetName().c_str(), src_graph->GetName().c_str()); + GELOGE(FAILED, "[Remove][JustNode] %s from graph:%s failed.", + input_node->GetName().c_str(), src_graph->GetName().c_str()); return FAILED; } // add input node to dst_graph if (dst_graph->AddNode(input_node) == nullptr) { - GELOGE(FAILED, "[GraphPartitioner]: AddNode() failed."); + REPORT_CALL_ERROR("E19999", "AddNode %s in graph:%s failed.", + input_node->GetName().c_str(), src_graph->GetName().c_str()); + GELOGE(FAILED, "[Add][Node] %s in graph:%s failed.", + input_node->GetName().c_str(), src_graph->GetName().c_str()); return FAILED; } if (LinkInput2EndRemoveOrginalLink(input_node, src_graph, dst_graph) != ge::SUCCESS) { - GELOGE(FAILED, "[GraphPartitioner]: LinkInput2EndRemoveOrginalLink() failed."); + GELOGE(FAILED, "[Call][LinkInput2EndRemoveOrginalLink] failed."); return FAILED; } } @@ -519,7 +621,7 @@ bool ge::GraphPartitioner::IsDataLike(ge::NodePtr node) { bool ge::GraphPartitioner::HasNoInput(ge::NodePtr node) { if (node == nullptr) { - GELOGE(FAILED, "node_ptr is null."); + GELOGE(FAILED, "[Check][Param] node is nullptr."); return true; } return node->GetInNodes().empty(); @@ -529,12 +631,13 @@ Status ge::GraphPartitioner::Initialize(ge::ComputeGraphPtr compute_graph) { GELOGI("Initialize starts."); std::shared_ptr instance_ptr = ge::GELib::GetInstance(); if (instance_ptr == nullptr || compute_graph == nullptr) { - GELOGE(GE_GRAPH_NOT_INIT, "Graph partitioner initialize failed."); + REPORT_INNER_ERROR("E19999", "compute_graph or instance_ptr of GELib is nullptr, check invalid."); + GELOGE(GE_GRAPH_NOT_INIT, "[Check][Param] compute_graph or instance_ptr of GELib is nullptr."); return FAILED; } graph_info_.engine_placer_.SetComputeGraph(compute_graph); if (graph_info_.engine_placer_.Run() != SUCCESS) { - GELOGE(FAILED, "Engine placer run failed."); + GELOGE(FAILED, "[Call][Run] Engine placer run failed, graph:%s.", compute_graph->GetName().c_str()); return FAILED; } const NodeEngineMap *node_engine_map = graph_info_.engine_placer_.GetNodeEngineMap(); @@ -552,14 +655,16 @@ Status ge::GraphPartitioner::Initialize(ge::ComputeGraphPtr compute_graph) { new_cluster = cluster; } else { if (node_engine_map->count(node) == 0) { - GELOGE(FAILED, "node[%s] does not owner engine!", node->GetName().c_str()); + REPORT_INNER_ERROR("E19999", "node:%s not find in node_engine_map", node->GetName().c_str()); + GELOGE(FAILED, "[Check][Param] node[%s] does not owner engine!", node->GetName().c_str()); return FAILED; } ClusterPtr cluster = MakeShared(temp_index, node_engine_map->at(node), temp_stream); new_cluster = cluster; } if (new_cluster == nullptr) { - GELOGE(FAILED, "[GraphPartitioner]: failed to allocate new_cluster"); + REPORT_CALL_ERROR("E19999", "Allocate Cluster failed, index:%zu", temp_index); + GELOGE(FAILED, "[Allocate][Cluster] failed, index:%zu", temp_index); return FAILED; } new_cluster->nodes_.push_back(node); @@ -570,11 +675,11 @@ Status ge::GraphPartitioner::Initialize(ge::ComputeGraphPtr compute_graph) { if (parent_id < node_id) { auto iter = graph_info_.node_2_cluster_.find(parent); if (iter == graph_info_.node_2_cluster_.end()) { - GELOGE(FAILED, - "[GraphPartitioner]: node[%s]id[%ld]'s parent_node[%s]id[%ld]" - "should make cluster in advance", - node->GetOpDesc()->GetName().c_str(), node_id, - parent->GetOpDesc()->GetName().c_str(), parent_id); + REPORT_INNER_ERROR("E19999", "node[%s]id[%ld]'s parent_node[%s]id[%ld] should make cluster in advance", + node->GetOpDesc()->GetName().c_str(), node_id, + parent->GetOpDesc()->GetName().c_str(), parent_id); + GELOGE(FAILED, "[Check][Param] node[%s]id[%ld]'s parent_node[%s]id[%ld] should make cluster in advance", + node->GetOpDesc()->GetName().c_str(), node_id, parent->GetOpDesc()->GetName().c_str(), parent_id); return FAILED; } new_cluster->in_clu_.insert(iter->second->index_); @@ -603,7 +708,8 @@ Status ge::GraphPartitioner::AddPartitionsToGraphNode(vectorGetName().c_str()); + GELOGE(GE_GRAPH_EMPTY_PARTITION, "[Check][Param] partition is null, subgraph:%s", sub_graph->GetName().c_str()); return FAILED; } auto &engine_name = graph_info_.partitions_.at(sub_graph); @@ -619,7 +725,8 @@ Status ge::GraphPartitioner::AddPartitionsToGraphNode(vectorSetParentNode(compute_graph->GetParentNode()); auto sgi = MakeShared(); if (sgi == nullptr) { - GELOGE(GE_GRAPH_PARAM_NULLPTR, "[GraphPartitioner]: MakeShared sub graph info failed."); + REPORT_CALL_ERROR("E19999", "allocate memory for SubGraphInfo failed."); + GELOGE(GE_GRAPH_PARAM_NULLPTR, "[Allocate][Memory] for SubGraphInfo failed."); return FAILED; } // set engine name @@ -767,7 +874,8 @@ void ge::GraphPartitioner::MarkClusters() { Status ge::GraphPartitioner::SplitSubGraphs(ge::ComputeGraphPtr compute_graph) { GELOGD("SplitSubGraphs starts."); if (compute_graph == nullptr) { - GELOGE(FAILED, "parameter ptr is null."); + REPORT_INNER_ERROR("E19999", "Param compute_graph is nullptr, check invalid"); + GELOGE(FAILED, "[Check][Param] parameter ptr is null."); return FAILED; } // Create graphs for all clusters @@ -785,7 +893,8 @@ Status ge::GraphPartitioner::SplitSubGraphs(ge::ComputeGraphPtr compute_graph) { std::string graph_name = "new_sub_graph" + std::to_string(graph_info_.partitions_.size()); ComputeGraphPtr new_sub_graph = MakeShared(graph_name); if (new_sub_graph == nullptr) { - GELOGE(GE_GRAPH_PARAM_NULLPTR, "[GraphPartitioner]: MakeShared() failed."); + REPORT_CALL_ERROR("E19999", "allocate memory for ge::ComputeGraph failed."); + GELOGE(GE_GRAPH_PARAM_NULLPTR, "[Allocate][Memory] for ge::ComputeGraph failed."); return FAILED; } AddNewGraphToPartition(new_sub_graph, child_cluster->engine_name_); @@ -796,7 +905,10 @@ Status ge::GraphPartitioner::SplitSubGraphs(ge::ComputeGraphPtr compute_graph) { // build node to corresponding node map NodePtr corresponding_node = corresponding_graph->AddNode(node->GetOpDesc()); if (corresponding_node == nullptr) { - GELOGE(GE_GRAPH_PARAM_NULLPTR, "[GraphPartitioner]: AddNode() failed."); + REPORT_CALL_ERROR("E19999", "add node:%s in graph:%s failed", + node->GetName().c_str(), corresponding_graph->GetName().c_str()); + GELOGE(GE_GRAPH_PARAM_NULLPTR, "[Add][Node] %s in graph:%s failed.", + node->GetName().c_str(), corresponding_graph->GetName().c_str()); return FAILED; } graph_info_.corresponding_node_in_partitions_[node] = corresponding_node; @@ -808,8 +920,12 @@ Status ge::GraphPartitioner::SplitSubGraphs(ge::ComputeGraphPtr compute_graph) { // Normally, all nodes have a copy in corresponding_node_in_partitions_, so function at can not be exception auto iter = graph_info_.corresponding_node_in_partitions_.find(peer_out_anchor->GetOwnerNode()); if (iter == graph_info_.corresponding_node_in_partitions_.end()) { - GELOGE(GRAPH_FAILED, - "[SpiltSubGraphs]: node[%s]id[%ld]'s parent_node[%s]id[%ld]" + REPORT_INNER_ERROR("E19999", "node[%s]id[%ld]'s parent_node[%s]id[%ld]" + "should make corresponding in advance", + node->GetOpDesc()->GetName().c_str(), node->GetOpDesc()->GetId(), + peer_out_anchor->GetOwnerNode()->GetOpDesc()->GetName().c_str(), + peer_out_anchor->GetOwnerNode()->GetOpDesc()->GetId()); + GELOGE(GRAPH_FAILED, "[Check][Param] node[%s]id[%ld]'s parent_node[%s]id[%ld]" "should make corresponding in advance", node->GetOpDesc()->GetName().c_str(), node->GetOpDesc()->GetId(), peer_out_anchor->GetOwnerNode()->GetOpDesc()->GetName().c_str(), @@ -827,13 +943,19 @@ Status ge::GraphPartitioner::SplitSubGraphs(ge::ComputeGraphPtr compute_graph) { if (parent_cluster != child_cluster) { GELOGD("Parent cluster is %zu, child_cluster is %zu", parent_cluster->index_, child_cluster->index_); if (AddPlaceHolderEnd(peer_out_anchor, in_anchor) != ge::SUCCESS) { - GELOGE(GE_GRAPH_ADD_PLC_END_FAILED, "[GraphPartitioner]: AddPlaceHolderEndInSrcDstGraph() failed."); + GELOGE(GE_GRAPH_ADD_PLC_END_FAILED, + "[Call][AddPlaceHolderEnd] failed, out_anchor:%s index:%d, in_anchor:%s index:%d.", + peer_out_anchor->GetOwnerNode()->GetName().c_str(), AnchorUtils::GetIdx(peer_out_anchor), + in_anchor->GetOwnerNode()->GetName().c_str(), AnchorUtils::GetIdx(in_anchor)); return FAILED; } } else { // parent and child in the same cluster, add edge GELOGD("AddEdge from parent cluster %zu to child %zu", parent_cluster->index_, child_cluster->index_); if (GraphUtils::AddEdge(src_anchor, dst_anchor) != GRAPH_SUCCESS) { - GELOGE(GRAPH_FAILED, "AddEdge fail, from %s to %s", peer_out_anchor->GetOwnerNode()->GetName().c_str(), + REPORT_CALL_ERROR("E19999", "add edge from %s to %s failed", + peer_out_anchor->GetOwnerNode()->GetName().c_str(), + in_anchor->GetOwnerNode()->GetName().c_str()); + GELOGE(GRAPH_FAILED, "[Add][Edge] from %s to %s failed", peer_out_anchor->GetOwnerNode()->GetName().c_str(), in_anchor->GetOwnerNode()->GetName().c_str()); return FAILED; } @@ -880,7 +1002,7 @@ Status ge::GraphPartitioner::Partition(ge::ComputeGraphPtr compute_graph, Mode m auto real_ret = SUCCESS; auto ret = PartitionSubGraph(compute_graph, mode); if (ret != SUCCESS) { - GELOGE(ret, "Sub graph partition Failed"); + GELOGE(ret, "[Partition][SubGraph] Failed, ret:%d", ret); real_ret = ret; } GE_CHECK_NOTNULL(compute_graph); @@ -888,7 +1010,7 @@ Status ge::GraphPartitioner::Partition(ge::ComputeGraphPtr compute_graph, Mode m for (const auto &sub_graph : compute_graph->GetAllSubgraphs()) { ret = PartitionSubGraph(sub_graph, mode); if (ret != SUCCESS) { - GELOGE(ret, "Sub graph partition Failed"); + GELOGE(ret, "[Partition][SubGraph] Failed, ret:%d", ret); real_ret = ret; } } @@ -903,7 +1025,8 @@ Status ge::GraphPartitioner::Partition(ge::ComputeGraphPtr compute_graph, Mode m Status ge::GraphPartitioner::PartitionSubGraph(ge::ComputeGraphPtr compute_graph, Mode mode) { if (compute_graph == nullptr) { - GELOGE(GE_GRAPH_NULL_INPUT, "[GraphPartitioner]: compute_graph is null."); + REPORT_INNER_ERROR("E19999", "Param compute_graph is nullptr, check invalid."); + GELOGE(GE_GRAPH_NULL_INPUT, "[Check][Param] compute_graph is nullptr."); return FAILED; } // clear graph_info @@ -912,18 +1035,24 @@ Status ge::GraphPartitioner::PartitionSubGraph(ge::ComputeGraphPtr compute_graph graph_info_.output_size_ = compute_graph->GetOutputSize(); graph_info_.input_size_ = compute_graph->GetInputSize(); if (graph_info_.output_size_ == 0) { - GELOGE(GE_GRAPH_NULL_INPUT, "The output size need to be greater than 0."); + REPORT_INNER_ERROR("E19999", "the output size of graph:%s is 0, check invalid.", + compute_graph->GetName().c_str()); + GELOGE(GE_GRAPH_NULL_INPUT, "[Check][Param] The output size:0 of graph:%s need to be greater than 0.", + compute_graph->GetName().c_str()); return FAILED; } GELOGI("Graph Partition starts, graph nodes size is %zu", compute_graph->GetDirectNodesSize()); Status ret = compute_graph->TopologicalSorting(); if (ret != SUCCESS) { - GELOGE(GE_GRAPH_TOPO_SORT_FAILED, "[GraphPartitioner]: subGraphPtr->TopologicalSorting failed"); + REPORT_CALL_ERROR("E19999", "TopologicalSorting for graph:%s failed", + compute_graph->GetName().c_str()); + GELOGE(GE_GRAPH_TOPO_SORT_FAILED, "[Call][TopologicalSorting] for subGraph:%s failed", + compute_graph->GetName().c_str()); return FAILED; } GE_TIMESTAMP_START(PartitionSubGraphInitialize); if (Initialize(compute_graph) != SUCCESS) { - GELOGE(GE_GRAPH_INIT_FAILED, "[GraphPartitioner]: initialize failed"); + GELOGE(GE_GRAPH_INIT_FAILED, "[Call][Initialize] for graph:%s failed", compute_graph->GetName().c_str()); return FAILED; } GE_TIMESTAMP_END(PartitionSubGraphInitialize, "GraphPartitioner::PartitionInitialize"); @@ -932,20 +1061,22 @@ Status ge::GraphPartitioner::PartitionSubGraph(ge::ComputeGraphPtr compute_graph GE_TIMESTAMP_END(PartitionSubGraphMarkClusters, "GraphPartitioner::PartitionMarkClusters"); GE_TIMESTAMP_START(PartitionSubGraphSplitSubGraphs); if (SplitSubGraphs(compute_graph) != SUCCESS) { - GELOGE(FAILED, "[GraphPartitioner]: SplitSubGraphs failed"); + GELOGE(FAILED, "[Split][SubGraphs] for graph:%s failed", compute_graph->GetName().c_str()); return FAILED; } GE_TIMESTAMP_END(PartitionSubGraphSplitSubGraphs, "GraphPartitioner::PartitionSplitSubGraphs"); GE_TIMESTAMP_START(PartitionSubGraphSortSubGraphs); if (SortSubGraphs(compute_graph) != ge::SUCCESS) { - GELOGE(GE_GRAPH_TOPO_SORT_FAILED, "Graph Partition SortSubGraphs failed."); + GELOGE(GE_GRAPH_TOPO_SORT_FAILED, "[Sort][SubGraphs] for graph:%s failed.", + compute_graph->GetName().c_str()); return ge::FAILED; } GE_TIMESTAMP_END(PartitionSubGraphSortSubGraphs, "GraphPartitioner::PartitionSortSubGraphs"); GE_TIMESTAMP_START(PartitionSubGraphAddPartitionsToGraphNode); vector output_subgraphs; if (AddPartitionsToGraphNode(output_subgraphs, compute_graph) != ge::SUCCESS) { - GELOGE(GE_GRAPH_EMPTY_PARTITION, "Graph Partition AddPartitionsToGraphNode failed."); + GELOGE(GE_GRAPH_EMPTY_PARTITION, "[Add][Partitions] To GraphNode failed, graph:%s.", + compute_graph->GetName().c_str()); return ge::FAILED; } GE_TIMESTAMP_END(PartitionSubGraphAddPartitionsToGraphNode, "GraphPartitioner::PartitionAddPartitionsToGraphNode"); @@ -961,14 +1092,16 @@ Status ge::GraphPartitioner::PartitionSubGraph(ge::ComputeGraphPtr compute_graph // all the inputs are the nodes and anchors in the original graph Status ge::GraphPartitioner::AddPlaceHolderEnd(const AnchorPtr &out_anchor, const AnchorPtr &in_anchor) { if ((out_anchor == nullptr) || (in_anchor == nullptr)) { - GELOGE(GE_GRAPH_PARAM_NULLPTR, "src_node or dst_node is null."); + REPORT_INNER_ERROR("E19999", "Param out_anchor or in_anchor is nullptr, check invalid."); + GELOGE(GE_GRAPH_PARAM_NULLPTR, "[Check][Param] out_anchor or in_anchor is nullptr."); return FAILED; } // nodes in original graph const auto &src_node = out_anchor->GetOwnerNode(); const auto &dst_node = in_anchor->GetOwnerNode(); if ((src_node == nullptr) || (dst_node == nullptr)) { - GELOGE(GE_GRAPH_PARAM_NULLPTR, "src_node or dst_node is null."); + REPORT_INNER_ERROR("E19999", "in_anchor'node or out_anchor'node is nullptr. check invalid."); + GELOGE(GE_GRAPH_PARAM_NULLPTR, "[Check][Param] src_node or dst_node is nullptr."); return FAILED; } // All nodes have a copy in corresponding_node_in_partitions_, so function at can not be execption @@ -977,7 +1110,10 @@ Status ge::GraphPartitioner::AddPlaceHolderEnd(const AnchorPtr &out_anchor, cons auto dst_anchor = graph_info_.corresponding_node_in_partitions_.at(dst_node)->GetInAnchor(AnchorUtils::GetIdx(in_anchor)); if ((src_anchor == nullptr) || (dst_anchor == nullptr)) { - GELOGE(GE_GRAPH_PARAM_NULLPTR, "src_anchor or dst_anchor is null."); + REPORT_INNER_ERROR("E19999", "src_anchor(index:%d) or dst_anchor(index:%d) is nullptr.", + AnchorUtils::GetIdx(out_anchor), AnchorUtils::GetIdx(in_anchor)); + GELOGE(GE_GRAPH_PARAM_NULLPTR, "[Check][Param] src_anchor(index:%d) or dst_anchor(index:%d) is nullptr.", + AnchorUtils::GetIdx(out_anchor), AnchorUtils::GetIdx(in_anchor)); return FAILED; } // anchors in subGraph @@ -986,7 +1122,7 @@ Status ge::GraphPartitioner::AddPlaceHolderEnd(const AnchorPtr &out_anchor, cons // add end and pld node auto ret = AddPlaceHolderEndInSrcDstGraph(src_anchor, dst_anchor, dst_subgraph, src_subgraph); if (ret != SUCCESS) { - GELOGE(GE_GRAPH_ADD_PLC_END_FAILED, "[GraphPartitioner]: add placeholder end failed."); + GELOGE(GE_GRAPH_ADD_PLC_END_FAILED, "[Call][AddPlaceHolderEndInSrcDstGraph] failed, ret:%d.", ret); return ret; } return SUCCESS; @@ -996,7 +1132,8 @@ Status ge::GraphPartitioner::SortSubGraphs(const ge::ComputeGraphPtr &compute_gr uint32_t rank = kRankOne; // rank 0 for data graph ComputeGraphPtr new_input_nodes_sub_graph = MakeShared("inputNodeGraph"); if ((new_input_nodes_sub_graph == nullptr) || (compute_graph == nullptr)) { - GELOGE(FAILED, "[GraphPartitioner]: new_input_nodes_sub_graph or compute_graph is null."); + REPORT_INNER_ERROR("E19999", "Param compute_graph is nullptr or Allocate Memory for ComputeGraph failed."); + GELOGE(FAILED, "[Check][Param] new_input_nodes_sub_graph or compute_graph is nullptr."); return FAILED; } for (const auto &node : compute_graph->GetDirectNode()) { @@ -1009,7 +1146,7 @@ Status ge::GraphPartitioner::SortSubGraphs(const ge::ComputeGraphPtr &compute_gr rank++; } else if (graph_info_.partitions_[sub_graph] == kEngineDefaultData) { // merge data graph if (PutInputNodesInSubGraph(sub_graph, new_input_nodes_sub_graph) != SUCCESS) { - GELOGE(FAILED, "[GraphPartitioner]: putInputNodesInSubGraph failed."); + GELOGE(FAILED, "[Call][putInputNodesInSubGraph] failed."); return FAILED; } auto to_be_del = graph_info_.partitions_.find(sub_graph); @@ -1038,7 +1175,8 @@ Status ge::GraphPartitioner::SortSubGraphs(const ge::ComputeGraphPtr &compute_gr AnchorPtr ge::GraphPartitioner::GetEndInAnchor(const AnchorPtr &src_anchor, const NodePtr &end_node) { if ((src_anchor == nullptr) || (end_node == nullptr)) { - GELOGE(FAILED, "parameter ptr is null."); + REPORT_INNER_ERROR("E19999", "Param src_anchor or end_node is nullptr, check invalid."); + GELOGE(FAILED, "[Check][Param] parameter src_anchor or end_node is nullptr."); return nullptr; } AnchorPtr end_in_anchor; @@ -1052,7 +1190,8 @@ AnchorPtr ge::GraphPartitioner::GetEndInAnchor(const AnchorPtr &src_anchor, cons AnchorPtr ge::GraphPartitioner::GetPldOutAnchor(const NodePtr &pld_node, const AnchorPtr &dst_anchor) { if ((pld_node == nullptr) || (dst_anchor == nullptr)) { - GELOGE(FAILED, "parameter ptr is null."); + REPORT_INNER_ERROR("E19999", "Param pld_node or dst_anchor is nullptr, check invalid."); + GELOGE(FAILED, "[Check][Param] parameter pld_node or dst_anchor is nullptr."); return nullptr; } AnchorPtr pld_out_anchor; @@ -1066,7 +1205,7 @@ AnchorPtr ge::GraphPartitioner::GetPldOutAnchor(const NodePtr &pld_node, const A void ge::GraphPartitioner::AddEndPldInformationToSubGraphInfo(ge::SubGraphInfoPtr &subgraph_info) { if (subgraph_info == nullptr) { - GELOGE(FAILED, "parameter ptr is null."); + GELOGE(FAILED, "[Check][Param] parameter subgraph_info is nullptr."); return; } auto subgraph = subgraph_info->GetSubGraph(); diff --git a/ge/graph/partition/stage_partition.cc b/ge/graph/partition/stage_partition.cc index 14776200..309e24c4 100644 --- a/ge/graph/partition/stage_partition.cc +++ b/ge/graph/partition/stage_partition.cc @@ -54,12 +54,12 @@ Status StagePartitioner::Partition() { GE_DUMP(root_graph_, "BeforeStagePartition"); if (SplitStageLevel() != SUCCESS) { - GELOGE(FAILED, "Split graph-stage for graph %s failed.", root_graph_->GetName().c_str()); + GELOGE(FAILED, "[Split][GraphStage] for graph %s failed.", root_graph_->GetName().c_str()); return FAILED; } if (StagePartition() != SUCCESS) { - GELOGE(FAILED, "Stage partition for graph %s failed.", root_graph_->GetName().c_str()); + GELOGE(FAILED, "[Stage][Partition] for graph %s failed.", root_graph_->GetName().c_str()); return FAILED; } @@ -71,7 +71,7 @@ Status StagePartitioner::Partition() { return a_level < b_level; }); if (root_graph_->TopologicalSorting() != GRAPH_SUCCESS) { - GELOGE(FAILED, "Topological sort for graph %s after stage partition failed, " + GELOGE(FAILED, "[Call][TopologicalSorting] for graph %s after stage partition failed, " "maybe stage_level was not set correctly.", root_graph_->GetName().c_str()); return FAILED; } @@ -103,7 +103,10 @@ Status StagePartitioner::SplitStageLevel() { continue; } if (!AttrUtils::SetInt(in_node->GetOpDesc(), ATTR_STAGE_LEVEL, cur_stage_level)) { - GELOGE(INTERNAL_ERROR, "Set attr ATTR_STAGE_LEVEL on node %s failed.", in_node->GetName().c_str()); + REPORT_CALL_ERROR("E19999", "Set Attr %s on node %s failed.", + ATTR_STAGE_LEVEL.c_str(), in_node->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Set][Attr] %s on node %s failed.", + ATTR_STAGE_LEVEL.c_str(), in_node->GetName().c_str()); return INTERNAL_ERROR; } GELOGD("Mark stage_level node %s, stage_level=%u", in_node->GetName().c_str(), cur_stage_level); @@ -131,23 +134,26 @@ Status StagePartitioner::StagePartition() { std::string subgraph_name = "Subgraph_Level_" + std::to_string(stage.first); NodePtr graph_node = BuildSubgraphNode(subgraph_name, stage_info); if (graph_node == nullptr) { - GELOGE(FAILED, "Build PartitionedCall node for stage %u failed.", stage.first); + GELOGE(FAILED, "[Build][SubgraphNode] for stage %u failed, graph name:%s.", stage.first, subgraph_name.c_str()); return FAILED; } ComputeGraphPtr subgraph = BuildStageGraph(graph_node, stage_info); if (subgraph == nullptr) { - GELOGE(FAILED, "Build subgraph for stage %u failed.", stage.first); + GELOGE(FAILED, "[Build][StageGraph] %s for stage %u failed.", graph_node->GetName().c_str(), stage.first); return FAILED; } if (root_graph_->AddSubgraph(subgraph) != GRAPH_SUCCESS) { - GELOGE(FAILED, "Add subgraph of stage %u failed.", stage.first); + REPORT_CALL_ERROR("E19999", "add subgraph:%s in root graph:%s of stage %u failed.", + subgraph->GetName().c_str(), root_graph_->GetName().c_str(), stage.first); + GELOGE(FAILED, "[Add][SubGraph] %s in root graph:%s of stage %u failed.", + subgraph->GetName().c_str(), root_graph_->GetName().c_str(), stage.first); return FAILED; } if ((RelinkDataEdges(graph_node, stage_info) != SUCCESS) || (RelinkCtrlEdges(graph_node, stage_info) != SUCCESS)) { - GELOGE(FAILED, "Relink edges for stage %u failed.", stage.first); + GELOGE(FAILED, "[ReLink][Edges] for stage %u failed, graph_node:%s.", stage.first, graph_node->GetName().c_str()); return FAILED; } @@ -214,7 +220,7 @@ NodePtr StagePartitioner::BuildSubgraphNode(const std::string &graph_name, const for (size_t i = 0; i < input_num; i++) { auto input_desc = stage_info.data_inputs[i].second->GetOwnerNode()->GetOpDesc(); if (input_desc == nullptr) { - GELOGE(PARAM_INVALID, "op_desc is null, node: %s", + GELOGE(PARAM_INVALID, "[Check][Param] op_desc is null, node:%s", stage_info.data_inputs[i].second->GetOwnerNode()->GetName().c_str()); return nullptr; } @@ -225,7 +231,7 @@ NodePtr StagePartitioner::BuildSubgraphNode(const std::string &graph_name, const for (size_t i = 0; i < output_num; i++) { auto output_desc = stage_info.data_outputs[i].first->GetOwnerNode()->GetOpDesc(); if (output_desc == nullptr) { - GELOGE(PARAM_INVALID, "op_desc is null, node: %s", + GELOGE(PARAM_INVALID, "[Check][Param] op_desc is null, node:%s", stage_info.data_outputs[i].first->GetOwnerNode()->GetName().c_str()); return nullptr; } @@ -235,7 +241,7 @@ NodePtr StagePartitioner::BuildSubgraphNode(const std::string &graph_name, const OpDescPtr op_desc = op_desc_builder.Build(); if (op_desc == nullptr) { - GELOGE(FAILED, "Create op_desc for subgraph node failed, name:%s.", graph_name.c_str()); + GELOGE(FAILED, "[Create][OpDesc] for subgraph node failed, name:%s.", graph_name.c_str()); return nullptr; } @@ -243,17 +249,23 @@ NodePtr StagePartitioner::BuildSubgraphNode(const std::string &graph_name, const op_desc->SetSubgraphInstanceName(0, graph_name); if (!AttrUtils::SetInt(op_desc, ATTR_STAGE_LEVEL, stage_info.stage_level)) { - GELOGE(INTERNAL_ERROR, "Set attr ATTR_STAGE_LEVEL on node %s failed", op_desc->GetName().c_str()); + REPORT_CALL_ERROR("E19999", "set attr %s on node %s failed", ATTR_STAGE_LEVEL.c_str(), op_desc->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Set][Attr] %s on node %s failed", ATTR_STAGE_LEVEL.c_str(), op_desc->GetName().c_str()); return nullptr; } NodePtr subgraph_node = root_graph_->AddNode(op_desc); if (subgraph_node == nullptr) { - GELOGE(FAILED, "Add node %s failed.", graph_name.c_str()); + REPORT_CALL_ERROR("E19999", "add node:%s in graph:%s failed.", + op_desc->GetName().c_str(), root_graph_->GetName().c_str()); + GELOGE(FAILED, "[Add][Node] %s in graph:%s failed.", op_desc->GetName().c_str(), root_graph_->GetName().c_str()); return nullptr; } if (subgraph_node->SetOwnerComputeGraph(root_graph_) != GRAPH_SUCCESS) { - GELOGE(FAILED, "Set owner graph for node %s failed.", subgraph_node->GetName().c_str()); + REPORT_CALL_ERROR("E19999", "SetOwnerComputeGraph for node %s failed, grpah:%s.", + subgraph_node->GetName().c_str(), root_graph_->GetName().c_str()); + GELOGE(FAILED, "[Set][OwnerGraph] for node %s failed, grpah:%s.", + subgraph_node->GetName().c_str(), root_graph_->GetName().c_str()); return nullptr; } @@ -314,11 +326,13 @@ ComputeGraphPtr StagePartitioner::BuildStageGraph(const NodePtr &subgraph_node, std::string error_msg; ComputeGraphPtr subgraph = graph_builder.Build(error_code, error_msg); if (subgraph == nullptr) { - GELOGE(error_code, "Build subgraph %s failed: %s.", subgraph_node->GetName().c_str(), error_msg.c_str()); + GELOGE(error_code, "[Build][Subgraph] %s failed:%s.", subgraph_node->GetName().c_str(), error_msg.c_str()); return nullptr; } if (!AttrUtils::SetInt(subgraph, ATTR_STAGE_LEVEL, stage_info.stage_level)) { - GELOGE(FAILED, "Set ATTR_STAGE_LEVEL on graph %s failed.", subgraph->GetName().c_str()); + REPORT_CALL_ERROR("E19999", "set attr %s on graph %s failed.", + ATTR_STAGE_LEVEL.c_str(), subgraph->GetName().c_str()); + GELOGE(FAILED, "[Set][Attr] %s on graph %s failed.", ATTR_STAGE_LEVEL.c_str(), subgraph->GetName().c_str()); return nullptr; } @@ -329,7 +343,12 @@ Status StagePartitioner::RelinkDataEdges(const NodePtr &subgraph_node, const Sta // in data nodes for (size_t i = 0; i < stage_info.data_inputs.size(); i++) { if (stage_info.data_inputs[i].first->Unlink(stage_info.data_inputs[i].second) != GRAPH_SUCCESS) { - GELOGE(INTERNAL_ERROR, "Remove data edge %s:%d->%s:%d failed.", + REPORT_CALL_ERROR("E19999", "remove data edge from %s:%d to %s:%d failed", + stage_info.data_inputs[i].first->GetOwnerNode()->GetName().c_str(), + stage_info.data_inputs[i].first->GetIdx(), + stage_info.data_inputs[i].second->GetOwnerNode()->GetName().c_str(), + stage_info.data_inputs[i].second->GetIdx()); + GELOGE(INTERNAL_ERROR, "[Remove][DataEdge] %s:%d->%s:%d failed.", stage_info.data_inputs[i].first->GetOwnerNode()->GetName().c_str(), stage_info.data_inputs[i].first->GetIdx(), stage_info.data_inputs[i].second->GetOwnerNode()->GetName().c_str(), @@ -337,7 +356,11 @@ Status StagePartitioner::RelinkDataEdges(const NodePtr &subgraph_node, const Sta return INTERNAL_ERROR; } if (stage_info.data_inputs[i].first->LinkTo(subgraph_node->GetInDataAnchor(i)) != GRAPH_SUCCESS) { - GELOGE(INTERNAL_ERROR, "Add data edge %s:%d->%s:%zu failed.", + REPORT_CALL_ERROR("E19999", "add data edge from %s:%d to %s:%zu failed.", + stage_info.data_inputs[i].first->GetOwnerNode()->GetName().c_str(), + stage_info.data_inputs[i].first->GetIdx(), + subgraph_node->GetName().c_str(), i); + GELOGE(INTERNAL_ERROR, "[Add][DataEdge] %s:%d->%s:%zu failed.", stage_info.data_inputs[i].first->GetOwnerNode()->GetName().c_str(), stage_info.data_inputs[i].first->GetIdx(), subgraph_node->GetName().c_str(), i); @@ -350,14 +373,20 @@ Status StagePartitioner::RelinkDataEdges(const NodePtr &subgraph_node, const Sta GE_CHECK_NOTNULL(out_data_anchor); for (const auto &peer_in_anchor : stage_info.data_outputs[i].second) { if (stage_info.data_outputs[i].first->Unlink(peer_in_anchor) != GRAPH_SUCCESS) { - GELOGE(INTERNAL_ERROR, "Remove data edge %s:%d->%s:%d failed.", + REPORT_CALL_ERROR("E19999", "Remove data edge from %s:%d to %s:%d failed.", + stage_info.data_outputs[i].first->GetOwnerNode()->GetName().c_str(), + stage_info.data_outputs[i].first->GetIdx(), + peer_in_anchor->GetOwnerNode()->GetName().c_str(), peer_in_anchor->GetIdx()); + GELOGE(INTERNAL_ERROR, "[Remove][DataEdge] %s:%d->%s:%d failed.", stage_info.data_outputs[i].first->GetOwnerNode()->GetName().c_str(), stage_info.data_outputs[i].first->GetIdx(), peer_in_anchor->GetOwnerNode()->GetName().c_str(), peer_in_anchor->GetIdx()); return INTERNAL_ERROR; } if (out_data_anchor->LinkTo(peer_in_anchor) != GRAPH_SUCCESS) { - GELOGE(INTERNAL_ERROR, "Add data edge %s:%zu->%s:%d failed.", subgraph_node->GetName().c_str(), i, + REPORT_CALL_ERROR("E19999", "Add data edge from %s:%zu to %s:%d failed.", subgraph_node->GetName().c_str(), i, + peer_in_anchor->GetOwnerNode()->GetName().c_str(), peer_in_anchor->GetIdx()); + GELOGE(INTERNAL_ERROR, "[Add][DataEdge] %s:%zu->%s:%d failed.", subgraph_node->GetName().c_str(), i, peer_in_anchor->GetOwnerNode()->GetName().c_str(), peer_in_anchor->GetIdx()); return INTERNAL_ERROR; } @@ -371,13 +400,18 @@ Status StagePartitioner::RelinkCtrlEdges(const NodePtr &subgraph_node, const Sta // in ctrl nodes for (const auto &ctrl_input : stage_info.ctrl_inputs) { if (ctrl_input.first->Unlink(ctrl_input.second) != GRAPH_SUCCESS) { - GELOGE(INTERNAL_ERROR, "Remove ctrl edge %s->%s failed.", + REPORT_CALL_ERROR("E19999", "Remove ctrl edge %s->%s failed.", + ctrl_input.first->GetOwnerNode()->GetName().c_str(), + ctrl_input.second->GetOwnerNode()->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Remove][CtrlEdge] %s->%s failed.", ctrl_input.first->GetOwnerNode()->GetName().c_str(), ctrl_input.second->GetOwnerNode()->GetName().c_str()); return INTERNAL_ERROR; } if (!ctrl_input.first->IsLinkedWith(subgraph_node->GetInControlAnchor())) { if (ctrl_input.first->LinkTo(subgraph_node->GetInControlAnchor()) != GRAPH_SUCCESS) { - GELOGE(INTERNAL_ERROR, "Add ctrl edge %s->%s failed.", + REPORT_CALL_ERROR("E19999", "Add ctrl edge %s->%s failed.", + ctrl_input.first->GetOwnerNode()->GetName().c_str(), subgraph_node->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Add][CtrlEdge] %s->%s failed.", ctrl_input.first->GetOwnerNode()->GetName().c_str(), subgraph_node->GetName().c_str()); return INTERNAL_ERROR; } @@ -386,14 +420,19 @@ Status StagePartitioner::RelinkCtrlEdges(const NodePtr &subgraph_node, const Sta // out ctrl nodes for (const auto &ctrl_output : stage_info.ctrl_outputs) { if (ctrl_output.first->Unlink(ctrl_output.second) != GRAPH_SUCCESS) { - GELOGE(INTERNAL_ERROR, "Remove ctrl edge %s->%s failed.", + REPORT_CALL_ERROR("E19999", "Remove ctrl edge %s->%s failed.", + ctrl_output.first->GetOwnerNode()->GetName().c_str(), + ctrl_output.second->GetOwnerNode()->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Remove][CtrlEdge] %s->%s failed.", ctrl_output.first->GetOwnerNode()->GetName().c_str(), ctrl_output.second->GetOwnerNode()->GetName().c_str()); return INTERNAL_ERROR; } if (!subgraph_node->GetOutControlAnchor()->IsLinkedWith(ctrl_output.second)) { if (subgraph_node->GetOutControlAnchor()->LinkTo(ctrl_output.second) != GRAPH_SUCCESS) { - GELOGE(INTERNAL_ERROR, "Add ctrl edge %s->%s failed.", + REPORT_CALL_ERROR("E19999", "Add ctrl edge %s->%s failed.", + subgraph_node->GetName().c_str(), ctrl_output.second->GetOwnerNode()->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Add][CtrlEdge] %s->%s failed.", subgraph_node->GetName().c_str(), ctrl_output.second->GetOwnerNode()->GetName().c_str()); return INTERNAL_ERROR; } diff --git a/ge/graph/passes/addn_pass.cc b/ge/graph/passes/addn_pass.cc index 3e2d3f06..6863a812 100644 --- a/ge/graph/passes/addn_pass.cc +++ b/ge/graph/passes/addn_pass.cc @@ -27,14 +27,14 @@ Status AddNPass::Run(NodePtr &node) { GELOGD("AddNPass running"); if (node == nullptr) { REPORT_INNER_ERROR("E19999", "Param node is nullptr, check invalid"); - GELOGE(PARAM_INVALID, "param [node] must not be null."); + GELOGE(PARAM_INVALID, "[Check][Param] param [node] must not be null."); return PARAM_INVALID; } if (node->GetType() == ADDN) { if (node->GetOpDesc() == nullptr) { REPORT_INNER_ERROR("E19999", "Param op_desc of node is nullptr, check invalid"); - GELOGE(PARAM_INVALID, "Param [node] op desc is null."); + GELOGE(PARAM_INVALID, "[Get][OpDesc] Param [node] op desc is null."); return PARAM_INVALID; } // AddN with single input can be optimized diff --git a/ge/graph/passes/aicpu_constant_folding_pass.cc b/ge/graph/passes/aicpu_constant_folding_pass.cc index b5a989c8..8fdb51a1 100644 --- a/ge/graph/passes/aicpu_constant_folding_pass.cc +++ b/ge/graph/passes/aicpu_constant_folding_pass.cc @@ -123,7 +123,7 @@ Status AicpuConstantFoldingPass::GetInputAddrs(const vector &w vector &input_addrs) { if (weight_vec.empty()) { REPORT_INNER_ERROR("E19999", "Param weight_vec is empty, check invalid"); - GELOGE(FAILED, "Weight is null"); + GELOGE(FAILED, "[Check][Param] Weight is null"); return FAILED; } for (const ConstGeTensorPtr &weight : weight_vec) { @@ -135,7 +135,7 @@ Status AicpuConstantFoldingPass::GetInputAddrs(const vector &w if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%zu, ret = 0x%X", weight->GetData().size(), rt_ret); - GELOGE(rt_ret, "rtMemcpy error"); + GELOGE(rt_ret, "[Call][RtMemcpy] failed, size:%zu, ret = 0x%X", weight->GetData().size(), rt_ret); GE_CHK_RT(rtFree(input_addr)); return FAILED; } @@ -150,7 +150,8 @@ Status AicpuConstantFoldingPass::GetOutputAddrs(const OpDescPtr &node_desc, vect if (node_desc->GetOutputsSize() == 0) { REPORT_INNER_ERROR("E19999", "Ouput desc size of op:%s(%s) is 0, check invalid", node_desc->GetName().c_str(), node_desc->GetType().c_str()); - GELOGE(FAILED, "Output size is 0 "); + GELOGE(FAILED, "[Get][OutputsSize] Ouput desc size of op:%s(%s) is 0", + node_desc->GetName().c_str(), node_desc->GetType().c_str()); return FAILED; } for (size_t i = 0; i < node_desc->GetOutputsSize(); ++i) { @@ -178,7 +179,7 @@ Status AicpuConstantFoldingPass::GenerateDataPtrInfo(const vector &out if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtMalloc failed, size:%lu, ret = 0x%X", result_summary.shape_data_size, rt_ret); - GELOGE(rt_ret, "rtMalloc error"); + GELOGE(rt_ret, "[Call][RtMalloc] failed, size:%lu, ret = 0x%X", result_summary.shape_data_size, rt_ret); GE_CHK_RT(rtFree(raw_data_addr)); return FAILED; } @@ -208,7 +209,7 @@ Status AicpuConstantFoldingPass::UpdateWorkSpaceAddr(string &task_info, STR_FWK_ // Update the workspace_addr if (task_info.empty()) { REPORT_INNER_ERROR("E19999", "Param task_info is empty, check invalid"); - GELOGE(FAILED, "task_info is empty "); + GELOGE(FAILED, "[Check][Param] task_info is empty "); return FAILED; } void *workspace_addr = nullptr; @@ -218,7 +219,7 @@ Status AicpuConstantFoldingPass::UpdateWorkSpaceAddr(string &task_info, STR_FWK_ if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%zu, ret = 0x%X", task_info.size(), rt_ret); - GELOGE(rt_ret, "rtMemcpy error"); + GELOGE(rt_ret, "[Call][RtMemcpy] failed, size:%zu, ret = 0x%X", task_info.size(), rt_ret); GE_CHK_RT(rtFree(workspace_addr)); return FAILED; } @@ -232,7 +233,7 @@ Status AicpuConstantFoldingPass::UpdateInputAndOutputAddr(const vector auto addrs_size = sizeof(uint64_t) * (io_addrs.size()); if (addrs_size <= 0) { REPORT_INNER_ERROR("E19999", "Param io_addrs size is 0, check invalid"); - GELOGE(FAILED, "addrs_size is less than 1 "); + GELOGE(FAILED, "[Check][Param] addrs_size is less than 1 "); return FAILED; } void *input_output_addr = nullptr; @@ -241,7 +242,7 @@ Status AicpuConstantFoldingPass::UpdateInputAndOutputAddr(const vector if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%zu, ret = 0x%X", addrs_size, rt_ret); - GELOGE(rt_ret, "rtMemcpy error"); + GELOGE(rt_ret, "[Call][RtMemcpy] failed, size:%zu, ret = 0x%X", addrs_size, rt_ret); GE_CHK_RT(rtFree(input_output_addr)); return FAILED; } @@ -264,12 +265,12 @@ Status AicpuConstantFoldingPass::UpdateSingleOpAddr(string &task_info, const vec Status ret = UpdateInputAndOutputAddr(io_addrs, task); if (ret != SUCCESS) { - GELOGE(ret, "UpdateInputAndOutputAddr error"); + GELOGE(ret, "[Update][InputAndOutputAddr] failed, ret:%d", ret); return ret; } ret = UpdateWorkSpaceAddr(task_info, task); if (ret != SUCCESS) { - GELOGE(ret, "UpdateWorkSpaceAddr error"); + GELOGE(ret, "[Update][WorkSpaceAddr] failed, ret:%d", ret); return ret; } return SUCCESS; @@ -299,7 +300,7 @@ Status AicpuConstantFoldingPass::UpdateMemCopyAddr(string &task_info, const vect if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%zu, ret = 0x%X", data_size, rt_ret); - GELOGE(rt_ret, "rtMemcpy error"); + GELOGE(rt_ret, "[Call][RtMemcpy] failed, size:%zu, ret = 0x%X", data_size, rt_ret); GE_CHK_RT(rtFree(input_addr_ptr)); return FAILED; } @@ -311,12 +312,12 @@ Status AicpuConstantFoldingPass::UpdateMemCopyAddr(string &task_info, const vect Status ret = UpdateInputAndOutputAddr(io_addrs, task); if (ret != SUCCESS) { - GELOGE(ret, "UpdateInputAndOutputAddr error"); + GELOGE(ret, "[Update][InputAndOutputAddr] failed, ret:%d", ret); return ret; } ret = UpdateWorkSpaceAddr(task_info, task); if (ret != SUCCESS) { - GELOGE(ret, "UpdateWorkSpaceAddr error"); + GELOGE(ret, "[Update][WorkSpaceAddr] failed, ret:%d", ret); return ret; } return SUCCESS; @@ -328,14 +329,14 @@ Status AicpuConstantFoldingPass::LaunchSingleOpRunTask(const NodePtr &node, cons auto instance_ptr = ge::GELib::GetInstance(); if (instance_ptr == nullptr || !instance_ptr->InitFlag()) { REPORT_INNER_ERROR("E19999", "GeLib is not init before, check invalid"); - GELOGE(GE_CLI_GE_NOT_INITIALIZED, "GE is not initialized"); + GELOGE(GE_CLI_GE_NOT_INITIALIZED, "[Check][Param] GE is not initialized"); return GE_CLI_GE_NOT_INITIALIZED; } auto kernel_builder = OpsKernelBuilderManager::Instance().GetOpsKernelBuilder(kKernelLibName); if (kernel_builder == nullptr) { REPORT_INNER_ERROR("E19999", "Find ops kernel by name:%s failed", kKernelLibName); - GELOGE(FAILED, "Get op kernel info store failed"); + GELOGE(FAILED, "[Get][OpsKernelBuilder] by name:%s failed", kKernelLibName); return FAILED; } STR_FWK_OP_KERNEL aicpu_task; @@ -364,17 +365,17 @@ Status AicpuConstantFoldingPass::LaunchSingleOpRunTask(const NodePtr &node, cons ret = UpdateSingleOpAddr(task_info, input_addrs, output_addrs, aicpu_task); if (ret != SUCCESS) { - GELOGE(ret, "UpdateSingleOpAddr error"); + GELOGE(ret, "[Update][SingleOpAddr] failed, ret:%d", ret); return ret; } ret = GenerateTaskForLaunch(aicpu_task, task_buf); if (ret != SUCCESS) { - GELOGE(ret, "GenerateTaskForLaunch error"); + GELOGE(ret, "[Generate][Task] For Launch failed, ret:%d", ret); return ret; } ret = KernelLaunch(task_buf); if (ret != SUCCESS) { - GELOGE(ret, "KernelLaunch error"); + GELOGE(ret, "[Call][KernelLaunch] failed, ret:%d", ret); return ret; } @@ -386,14 +387,14 @@ Status AicpuConstantFoldingPass::LaunchMemCopyTask(const vector &data_ auto instance_ptr = ge::GELib::GetInstance(); if (instance_ptr == nullptr || !instance_ptr->InitFlag()) { REPORT_INNER_ERROR("E19999", "GeLib is not init before, check invalid"); - GELOGE(GE_CLI_GE_NOT_INITIALIZED, "GE is not initialized"); + GELOGE(GE_CLI_GE_NOT_INITIALIZED, "[Check][Param] GE is not initialized"); return GE_CLI_GE_NOT_INITIALIZED; } auto kernel_builder = OpsKernelBuilderManager::Instance().GetOpsKernelBuilder(kKernelLibName); if (kernel_builder == nullptr) { REPORT_INNER_ERROR("E19999", "Find ops kernel by name:%s failed", kKernelLibName); - GELOGE(FAILED, "Get op kernel info store failed"); + GELOGE(FAILED, "[Get][OpsKernelBuilder] by name:%s failed", kKernelLibName); return FAILED; } STR_FWK_OP_KERNEL aicpu_task; @@ -427,17 +428,17 @@ Status AicpuConstantFoldingPass::LaunchMemCopyTask(const vector &data_ ret = UpdateMemCopyAddr(task_info, data_infos, internal_addrs, aicpu_task); if (ret != SUCCESS) { - GELOGE(ret, "UpdateMemCopyAddr error"); + GELOGE(ret, "[Update][MemCopyAddr] failed, ret:%d", ret); return ret; } ret = GenerateTaskForLaunch(aicpu_task, task_buf); if (ret != SUCCESS) { - GELOGE(ret, "GenerateTaskForLaunch error"); + GELOGE(ret, "[Generate][Task] For Launch failed, ret:%d", ret); return ret; } ret = KernelLaunch(task_buf); if (ret != SUCCESS) { - GELOGE(ret, "KernelLaunch error"); + GELOGE(ret, "[Call][KernelLaunch] failed, ret:%d", ret); return ret; } return SUCCESS; @@ -451,7 +452,7 @@ Status AicpuConstantFoldingPass::GenerateTaskForLaunch(STR_FWK_OP_KERNEL &aicpu_ if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtMemcpy failed, size:%zu, ret = 0x%X", sizeof(STR_FWK_OP_KERNEL), rt_ret); - GELOGE(rt_ret, "rtMemcpy error"); + GELOGE(rt_ret, "[Call][RtMemcpy] failed, size:%zu, ret = 0x%X", sizeof(STR_FWK_OP_KERNEL), rt_ret); GE_CHK_RT(rtFree(task_buf)); return FAILED; } @@ -482,56 +483,56 @@ Status AicpuConstantFoldingPass::KernelLaunch(void *task_buf) { if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtModelCreate failed, ret = 0x%X", rt_ret); - GELOGE(rt_ret, "create model failed."); + GELOGE(rt_ret, "[Create][Model] failed, ret = 0x%X", rt_ret); return FAILED; } rt_ret = rtStreamCreate(&stream, 0); if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtStreamCreate failed, ret = 0x%X", rt_ret); - GELOGE(rt_ret, "create stream failed."); + GELOGE(rt_ret, "[Create][Stream] failed, ret = 0x%X", rt_ret); return FAILED; } rt_ret = rtModelBindStream(model, stream, 0); if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtModelBindStream failed, ret = 0x%X", rt_ret); - GELOGE(rt_ret, "rtModelBindStream failed."); + GELOGE(rt_ret, "[Call][RtModelBindStream] failed, ret = 0x%X", rt_ret); return FAILED; } rt_ret = rtKernelLaunchEx(task_buf, sizeof(STR_FWK_OP_KERNEL), 0, stream); if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtModelBindStream failed, ret = 0x%X", rt_ret); - GELOGE(rt_ret, "rtKernelLaunchEx failed."); + GELOGE(rt_ret, "[Call][RtModelBindStream] failed, ret = 0x%X", rt_ret); return FAILED; } rt_ret = rtModelLoadComplete(model); if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtModelLoadComplete failed, ret = 0x%X", rt_ret); - GELOGE(rt_ret, "rtModelLoadComplete failed."); + GELOGE(rt_ret, "[Call][RtModelLoadComplete] failed, ret = 0x%X", rt_ret); return FAILED; } rt_ret = rtStreamCreate(&stream_run, 0); if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtStreamCreate failed, ret = 0x%X", rt_ret); - GELOGE(rt_ret, "create run stream failed."); + GELOGE(rt_ret, "[Call][RtStreamCreate] failed, ret = 0x%X", rt_ret); return FAILED; } rt_ret = rtModelExecute(model, stream_run, 0); if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtModelExecute failed, ret = 0x%X", rt_ret); - GELOGE(rt_ret, "rtModelExecute failed."); + GELOGE(rt_ret, "[Call][RtModelExecute] failed, ret = 0x%X", rt_ret); return FAILED; } rt_ret = rtStreamSynchronize(stream_run); if (rt_ret != RT_ERROR_NONE) { REPORT_CALL_ERROR("E19999", "Call rtStreamSynchronize failed, ret = 0x%X", rt_ret); - GELOGE(rt_ret, "rtStreamSynchronize failed."); + GELOGE(rt_ret, "[Call][RtStreamSynchronize] failed, ret = 0x%X", rt_ret); return FAILED; } return SUCCESS; @@ -543,7 +544,8 @@ Status AicpuConstantFoldingPass::GenerateGeTensor(const OpDescPtr &node_desc, co REPORT_INNER_ERROR("E19999", "Output desc size:%zu of op:%s(%s), after multi 2, not equal to data_vec.size:%zu, " "check invalid", node_desc->GetOutputsSize(), node_desc->GetName().c_str(), node_desc->GetType().c_str(), data_vec.size()); - GELOGE(FAILED, "node[%s] something wrong with output size", node_desc->GetName().c_str()); + GELOGE(FAILED, "[Check][Param] Output desc size:%zu of op:%s(%s), after multi 2, not equal to data_vec.size:%zu", + node_desc->GetOutputsSize(), node_desc->GetName().c_str(), node_desc->GetType().c_str(), data_vec.size()); return FAILED; } @@ -552,7 +554,7 @@ Status AicpuConstantFoldingPass::GenerateGeTensor(const OpDescPtr &node_desc, co GeTensorPtr output_ptr = MakeShared(output_tensor_desc); if (output_ptr == nullptr) { REPORT_CALL_ERROR("E19999", "New GeTensor failed"); - GELOGE(FAILED, "node[%s] something wrong with construct GeTensor", node_desc->GetName().c_str()); + GELOGE(FAILED, "[New][GeTensor] failed"); return FAILED; } const DataPtrInfo &raw_data_info = data_vec.at(i * kDouble); @@ -561,14 +563,14 @@ Status AicpuConstantFoldingPass::GenerateGeTensor(const OpDescPtr &node_desc, co if (data_addr == nullptr) { REPORT_CALL_ERROR("E19999", "New Buffer failed, size:%lu", raw_data_size); - GELOGE(MEMALLOC_FAILED, "new data_addr failed"); + GELOGE(MEMALLOC_FAILED, "[New][Buffer] failed, size:%lu", raw_data_size); return INTERNAL_ERROR; } GE_CHK_RT_RET(rtMemcpy(data_addr.get(), raw_data_size, reinterpret_cast(reinterpret_cast(raw_data_info.dst_ptr)), raw_data_size, RT_MEMCPY_DEVICE_TO_HOST)); GE_IF_BOOL_EXEC(output_ptr->SetData(data_addr.get(), raw_data_size) != GRAPH_SUCCESS, - GELOGE(FAILED, "set data failed"); + GELOGE(FAILED, "[Set][Data] for node:%s output[%zu] failed", node_desc->GetName().c_str(), i); return FAILED); GELOGD("GenerateGeTensor: raw_data_size %lu", raw_data_size); @@ -586,7 +588,7 @@ Status AicpuConstantFoldingPass::GenerateGeTensor(const OpDescPtr &node_desc, co if (shape_addr == nullptr) { REPORT_CALL_ERROR("E19999", "New Buffer failed, size:%lu", dim_num); - GELOGE(MEMALLOC_FAILED, "new shape_addr failed"); + GELOGE(MEMALLOC_FAILED, "[New][Buffer] failed, size:%lu", dim_num); return INTERNAL_ERROR; } GE_CHK_RT_RET(rtMemcpy(shape_addr.get(), shape_data_size, @@ -632,22 +634,23 @@ bool AicpuConstantFoldingPass::IsSkipFold(const ge::NodePtr &node) { auto instance_ptr = ge::GELib::GetInstance(); if (instance_ptr == nullptr || !instance_ptr->InitFlag()) { REPORT_INNER_ERROR("E19999", "GeLib is not init before, check invalid"); - GELOGE(GE_CLI_GE_NOT_INITIALIZED, "GE is not initialized"); + GELOGE(GE_CLI_GE_NOT_INITIALIZED, "[Check][Param] GE is not initialized"); return true; } OpsKernelInfoStorePtr kernel_info = instance_ptr->OpsKernelManagerObj().GetOpsKernelInfoStore(kKernelLibName); if (kernel_info == nullptr) { REPORT_INNER_ERROR("E19999", "Find ops kernel by name:%s failed", kKernelLibName); - GELOGE(FAILED, "Get op kernel info store failed"); + GELOGE(FAILED, "[Get][OpsKernelInfoStore] by name:%s failed", kKernelLibName); return true; } std::string check_result; kernel_info->opsFlagCheck(*node, check_result); if (check_result.empty()) { - REPORT_CALL_ERROR("E19999", "Call opsFlagCheck faled, ops kernel name:%s, op:%s(%s)", + REPORT_CALL_ERROR("E19999", "Call opsFlagCheck failed, ops kernel name:%s, op:%s(%s)", kKernelLibName, node->GetName().c_str(), node->GetType().c_str()); - GELOGE(FAILED, "Get op check_result failed"); + GELOGE(FAILED, "[Call][OpsFlagCheck] failed, ops kernel name:%s, op:%s(%s)", + kKernelLibName, node->GetName().c_str(), node->GetType().c_str()); return true; } return check_result.substr(0, kOpsFlag) == kNotSupported; diff --git a/ge/graph/passes/assert_pass.cc b/ge/graph/passes/assert_pass.cc index 20734d56..e13fc300 100644 --- a/ge/graph/passes/assert_pass.cc +++ b/ge/graph/passes/assert_pass.cc @@ -36,7 +36,7 @@ Status AssertPass::Run(NodePtr &node) { } if (node->GetOpDesc() == nullptr) { REPORT_INNER_ERROR("E19999", "Param op_desc of node is nullptr, check invalid"); - GELOGE(PARAM_INVALID, "param [node] [opDesc] must not be null."); + GELOGE(PARAM_INVALID, "[Get][OpDesc] param [node] [opDesc] must not be null."); return PARAM_INVALID; } std::string op_type = node->GetOpDesc()->GetType(); @@ -49,7 +49,7 @@ Status AssertPass::Run(NodePtr &node) { // remove unused node Status status = RemoveUnusedNode(nodes_unused); if (status != SUCCESS) { - GELOGE(status, "remove unused node failed."); + GELOGE(status, "[Remove][UnusedNode] failed, ret:%d.", status); return status; } } @@ -95,8 +95,10 @@ Status AssertPass::RemoveUnusedNode(std::vector &nodes_unused) { } if (IsolateAndDeleteNode(node, assert_io_map) != SUCCESS) { - REPORT_INNER_ERROR("E19999", "Isolate and delete node:%s(%s) faild", - node->GetName().c_str(), node->GetType().c_str()); + REPORT_CALL_ERROR("E19999", "Isolate and delete node:%s(%s) failed", + node->GetName().c_str(), node->GetType().c_str()); + GELOGE(FAILED, "[Call][IsolateAndDeleteNode] for node:%s(%s) failed", + node->GetName().c_str(), node->GetType().c_str()); return FAILED; } } diff --git a/ge/graph/passes/assign_remove_pass.cc b/ge/graph/passes/assign_remove_pass.cc index 43a95516..1d78d6bd 100644 --- a/ge/graph/passes/assign_remove_pass.cc +++ b/ge/graph/passes/assign_remove_pass.cc @@ -33,13 +33,13 @@ Status AssignRemovePass::Run(NodePtr &node) { GELOGD("AssignRemovePass running"); if (TransformAttr(node) != SUCCESS) { - GELOGE(FAILED, "Transform assign_var_name attr failed, node=%s", node->GetName().c_str()); + GELOGE(FAILED, "[Call][TransformAttr] Transform assign_var_name attr failed, node=%s", node->GetName().c_str()); return FAILED; } if (node->GetType() == ASSIGN) { if (OptimizedAssignNode(node) != SUCCESS) { - GELOGE(FAILED, "Optimize for assign_node %s failed", node->GetName().c_str()); + GELOGE(FAILED, "[Call][Optimize] for assign_node %s failed", node->GetName().c_str()); return FAILED; } } @@ -60,7 +60,9 @@ Status AssignRemovePass::OptimizedAssignNode(NodePtr &assign_node) { REPORT_INNER_ERROR("E19999", "Index %d or %d input anchor of node:%s(%s) is nullptr, check invalid", kAssignRefInputIndex, kAssignValueInputIndex, assign_node->GetName().c_str(), assign_node->GetType().c_str()); - GELOGE(FAILED, "In data anchor is null, node:%s", assign_node->GetName().c_str()); + GELOGE(FAILED, "[Check][Param] Index %d or %d input anchor of node:%s(%s) is nullptr", + kAssignRefInputIndex, kAssignValueInputIndex, + assign_node->GetName().c_str(), assign_node->GetType().c_str()); return FAILED; } const auto &ref_peer_anchor = ref_in_anchor->GetPeerOutAnchor(); @@ -69,7 +71,9 @@ Status AssignRemovePass::OptimizedAssignNode(NodePtr &assign_node) { REPORT_INNER_ERROR("E19999", "Index %d or %d input anchor of node:%s(%s), peer anchor is nullptr, check invalid", kAssignRefInputIndex, kAssignValueInputIndex, assign_node->GetName().c_str(), assign_node->GetType().c_str()); - GELOGE(FAILED, "Peer data anchor is null, node:%s", assign_node->GetName().c_str()); + GELOGE(FAILED, "[Check][Param] Index %d or %d input anchor of node:%s(%s), peer anchor is nullptr", + kAssignRefInputIndex, kAssignValueInputIndex, + assign_node->GetName().c_str(), assign_node->GetType().c_str()); return FAILED; } @@ -87,7 +91,7 @@ Status AssignRemovePass::OptimizedAssignNode(NodePtr &assign_node) { if (IsolateAndDeleteNode(assign_node, {kAssignRefInputIndex}) != SUCCESS) { REPORT_CALL_ERROR("E19999", "Isolate and delete node:%s(%s) failed", assign_node->GetName().c_str(), assign_node->GetType().c_str()); - GELOGE(FAILED, "Isolate and delete assign_node %s failed.", assign_node->GetName().c_str()); + GELOGE(FAILED, "[IsolateAndDelete][Node] %s failed.", assign_node->GetName().c_str()); return FAILED; } @@ -97,7 +101,9 @@ Status AssignRemovePass::OptimizedAssignNode(NodePtr &assign_node) { REPORT_INNER_ERROR("E19999", "Input index %d or %d of node:%s(%s), peer op is nullptr, check invalid", kAssignRefInputIndex, kAssignValueInputIndex, assign_node->GetName().c_str(), assign_node->GetType().c_str()); - GELOGE(FAILED, "value input is null"); + GELOGE(FAILED, "[Check][Param] Input index %d or %d of node:%s(%s), peer op is nullptr", + kAssignRefInputIndex, kAssignValueInputIndex, + assign_node->GetName().c_str(), assign_node->GetType().c_str()); return FAILED; } @@ -105,7 +111,8 @@ Status AssignRemovePass::OptimizedAssignNode(NodePtr &assign_node) { if (ref_input->UpdateInputDesc(0, value_input->GetOutputDesc(value_peer_anchor->GetIdx())) != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Input index %d of node:%s(%s), update it's peer op input:0 desc failed", kAssignRefInputIndex, assign_node->GetName().c_str(), assign_node->GetType().c_str()); - GELOGE(FAILED, "Update input_desc for variable %s failed.", ref_input->GetName().c_str()); + GELOGE(FAILED, "[Update][InputDesc] Input index %d of node:%s(%s), update it's peer op input:0 desc failed", + kAssignRefInputIndex, assign_node->GetName().c_str(), assign_node->GetType().c_str()); return FAILED; } if (GraphUtils::AddEdge(value_peer_anchor, ref_peer_anchor->GetOwnerNode()->GetInDataAnchor(0)) != GRAPH_SUCCESS) { @@ -114,7 +121,10 @@ Status AssignRemovePass::OptimizedAssignNode(NodePtr &assign_node) { value_peer_anchor->GetOwnerNode()->GetType().c_str(), value_peer_anchor->GetIdx(), ref_peer_anchor->GetOwnerNode()->GetName().c_str(), ref_peer_anchor->GetOwnerNode()->GetType().c_str()); - GELOGE(FAILED, "Add data edge %s->%s failed", value_input->GetName().c_str(), ref_input->GetName().c_str()); + GELOGE(FAILED, "[Add][Edge] between op:%s(%s)(out_index:%d) and op:%s(%s)(in_index:0) failed", + value_peer_anchor->GetOwnerNode()->GetName().c_str(), + value_peer_anchor->GetOwnerNode()->GetType().c_str(), value_peer_anchor->GetIdx(), + ref_peer_anchor->GetOwnerNode()->GetName().c_str(), ref_peer_anchor->GetOwnerNode()->GetType().c_str()); return FAILED; } @@ -125,7 +135,9 @@ Status AssignRemovePass::OptimizedAssignNode(NodePtr &assign_node) { REPORT_CALL_ERROR("E19999", "Set Attr:%s to output:%d desc of node:%s(%s) failed", ASSIGN_VAR_NAME.c_str(), value_peer_anchor->GetIdx(), value_input->GetName().c_str(), value_input->GetType().c_str()); - GELOGE(FAILED, "Set attr ASSIGN_VAR_NAME failed."); + GELOGE(FAILED, "[Set][Attr] %s to output:%d desc of node:%s(%s) failed", + ASSIGN_VAR_NAME.c_str(), value_peer_anchor->GetIdx(), + value_input->GetName().c_str(), value_input->GetType().c_str()); return FAILED; } auto value_node = value_peer_anchor->GetOwnerNode(); @@ -160,7 +172,9 @@ Status AssignRemovePass::TransformAttr(NodePtr &node) { REPORT_CALL_ERROR("E19999", "Set Attr:%s to output:%d desc of node:%s(%s) failed", ASSIGN_VAR_NAME.c_str(), peer_data_anchor->GetIdx(), in_node->GetName().c_str(), in_node->GetType().c_str()); - GELOGE(FAILED, "Set attr ASSIGN_VAR_NAME failed."); + GELOGE(FAILED, "[Set][Attr] %s to output:%d desc of node:%s(%s) failed", + ASSIGN_VAR_NAME.c_str(), peer_data_anchor->GetIdx(), + in_node->GetName().c_str(), in_node->GetType().c_str()); return FAILED; } AddRePassNode(in_node); diff --git a/ge/graph/passes/atomic_addr_clean_pass.cc b/ge/graph/passes/atomic_addr_clean_pass.cc index 1bc6e9c9..9f202c77 100755 --- a/ge/graph/passes/atomic_addr_clean_pass.cc +++ b/ge/graph/passes/atomic_addr_clean_pass.cc @@ -159,7 +159,8 @@ Status AtomicAddrCleanPass::HandleLoopGraph(ComputeGraphPtr &graph, const vector // Insert atomic clean op NodePtr clean_addr_node = InsertAtomicAddrCleanNode(graph); if (clean_addr_node == nullptr) { - GELOGE(FAILED, "Insert AtomicAddrClean node failed. Ignore atomicAddrClean pass."); + GELOGE(FAILED, "[Insert][AtomicAddrCleanNode] to graph:%s failed. Ignore atomicAddrClean pass.", + graph->GetName().c_str()); return FAILED; } @@ -173,7 +174,9 @@ Status AtomicAddrCleanPass::HandleLoopGraph(ComputeGraphPtr &graph, const vector auto ret = LinkToAtomicNode(node, clean_addr_node); if (ret != SUCCESS) { - GELOGE(ret, "Link control anchor failed from atomic node to atomic_addr_clean node."); + GELOGE(ret, + "[Call][LinkToAtomicNode] Link control anchor failed from atomic node:%s to atomic_addr_clean:%s node.", + node->GetName().c_str(), clean_addr_node->GetName().c_str()); return ret; } index++; @@ -188,7 +191,7 @@ Status AtomicAddrCleanPass::HandleNormalGraph(ComputeGraphPtr &graph, const vect vector dispersed_atomic_nodes; auto ret = HandleDispersedAtomicNodes(graph, atomic_node_vec, common_atomic_nodes, dispersed_atomic_nodes); if (ret != SUCCESS) { - GELOGE(ret, "Handle dispersed atomic nodes failed, graph name is %s.", graph->GetName().c_str()); + GELOGE(ret, "[Call][HandleDispersedAtomicNodes] failed, graph name is %s.", graph->GetName().c_str()); return ret; } @@ -200,13 +203,15 @@ Status AtomicAddrCleanPass::HandleNormalGraph(ComputeGraphPtr &graph, const vect // not loop graph , insert only one clean node in graph NodePtr clean_addr_node = InsertAtomicAddrCleanNode(graph); if (clean_addr_node == nullptr) { - GELOGE(FAILED, "Insert AtomicAddrClean node failed. Ignore atomicAddrClean pass."); + GELOGE(FAILED, "[Insert][AtomicAddrCleanNode] in graph:%s failed. Ignore atomicAddrClean pass.", + graph->GetName().c_str()); return FAILED; } for (const auto &node : common_atomic_nodes) { ret = LinkToAtomicNode(node, clean_addr_node); if (ret != SUCCESS) { - GELOGE(ret, "Link control anchor failed from atomic node to atomic_addr_clean node."); + GELOGE(ret, "[Link][ControlAnchor] failed from atomic node:%s to atomic_addr_clean node:%s.", + node->GetName().c_str(), clean_addr_node->GetName().c_str()); return ret; } } @@ -218,7 +223,8 @@ Status AtomicAddrCleanPass::HandleNormalGraph(ComputeGraphPtr &graph, const vect NodePtr peer_in_node = in_anchor->GetPeerOutAnchor()->GetOwnerNode(); ret = LinkToAtomicNode(peer_in_node, clean_addr_node); if (ret != SUCCESS) { - GELOGE(ret, "Link failed, %s : %s", peer_in_node->GetName().c_str(), clean_addr_node->GetName().c_str()); + GELOGE(ret, "[Link][ControlAnchor] from node:%s to node:%s failed", + peer_in_node->GetName().c_str(), clean_addr_node->GetName().c_str()); return ret; } } @@ -276,7 +282,8 @@ Status AtomicAddrCleanPass::HandleDispersedAtomicNodes(ComputeGraphPtr &graph, if (!node_anchors_connect_netoutput.empty()) { NodePtr dispersed_clean_addr_node = InsertAtomicAddrCleanNode(graph); if (dispersed_clean_addr_node == nullptr) { - GELOGE(FAILED, "Insert AtomicAddrClean node failed. Ignore atomicAddrClean pass."); + GELOGE(FAILED, "[Insert][AtomicAddrCleanNode] in graph:%s failed. Ignore atomicAddrClean pass.", + graph->GetName().c_str()); return FAILED; } @@ -291,7 +298,7 @@ Status AtomicAddrCleanPass::HandleDispersedAtomicNodes(ComputeGraphPtr &graph, ++index; Status ret = LinkToAtomicNode(node, dispersed_clean_addr_node); if (ret != SUCCESS) { - GELOGE(ret, "Link control anchor failed from atomic node: %s to atomic_addr_clean node: %s.", + GELOGE(ret, "[Link][ControlAnchor] failed from atomic node:%s to atomic_addr_clean node:%s.", node->GetName().c_str(), dispersed_clean_addr_node->GetName().c_str()); return ret; } @@ -308,7 +315,7 @@ NodePtr AtomicAddrCleanPass::InsertAtomicAddrCleanNode(ComputeGraphPtr &graph) { OpDescPtr op_desc = MakeShared(NODE_NAME_ATOMIC_ADDR_CLEAN, ATOMICADDRCLEAN); if (op_desc == nullptr) { REPORT_CALL_ERROR("E19999", "New OpDesc failed"); - GELOGE(INTERNAL_ERROR, "Make shared atomic addr clean op failed."); + GELOGE(INTERNAL_ERROR, "[New][OpDesc] failed."); return nullptr; } string session_graph_id; @@ -336,7 +343,7 @@ Status AtomicAddrCleanPass::LinkToAtomicNode(const NodePtr &atomic_node, NodePtr GE_IF_BOOL_EXEC(atomic_node == nullptr || atomic_clean_node == nullptr, REPORT_INNER_ERROR("E19999", "Param atomic_node or atomic_clean_node is nullptr, " "check invalid"); - DOMI_LOGE("param [atomic_node][atomic_clean_node] must not be null."); + DOMI_LOGE("[Check][Param] param [atomic_node][atomic_clean_node] must not be null."); return PARAM_INVALID); InControlAnchorPtr in_ctrl_anchor = atomic_node->GetInControlAnchor(); OutControlAnchorPtr out_ctrl_anchor = atomic_clean_node->GetOutControlAnchor(); @@ -345,9 +352,9 @@ Status AtomicAddrCleanPass::LinkToAtomicNode(const NodePtr &atomic_node, NodePtr "check invalid", atomic_node->GetName().c_str(), atomic_node->GetType().c_str(), atomic_clean_node->GetName().c_str(), atomic_clean_node->GetType().c_str()); - GELOGE(INTERNAL_ERROR, - "Get control anchor faild, dst node: %s.", - atomic_node->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Check][Param] in_ctrl_anchor of op:%s(%s) or out_ctrl_anchor of op:%s(%s) is nullptr.", + atomic_node->GetName().c_str(), atomic_node->GetType().c_str(), + atomic_clean_node->GetName().c_str(), atomic_clean_node->GetType().c_str()); return INTERNAL_ERROR; } @@ -358,9 +365,9 @@ Status AtomicAddrCleanPass::LinkToAtomicNode(const NodePtr &atomic_node, NodePtr out_ctrl_anchor->GetOwnerNode()->GetType().c_str(), in_ctrl_anchor->GetOwnerNode()->GetName().c_str(), in_ctrl_anchor->GetOwnerNode()->GetType().c_str()); - GELOGE(INTERNAL_ERROR, - "Graph add cleanAddrNode op out ctrl edge fail, dst node: %s.", - atomic_node->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Add][ControlEdge] between op:%s(%s) and op:%s(%s) failed", + out_ctrl_anchor->GetOwnerNode()->GetName().c_str(), out_ctrl_anchor->GetOwnerNode()->GetType().c_str(), + in_ctrl_anchor->GetOwnerNode()->GetName().c_str(), in_ctrl_anchor->GetOwnerNode()->GetType().c_str()); return INTERNAL_ERROR; } GELOGD("Graph add cleanAddrNode op out ctrl edge, dst node: %s.", atomic_node->GetName().c_str()); @@ -375,7 +382,7 @@ Status AtomicAddrCleanPass::LinkToAtomicNode(const NodePtr &atomic_node, NodePtr } bool AtomicAddrCleanPass::IsAtomicOp(const NodePtr &node) { - GE_IF_BOOL_EXEC(node == nullptr, GELOGE(FAILED, "node is null."); return false); + GE_IF_BOOL_EXEC(node == nullptr, GELOGE(FAILED, "[Check][Param] node is nullptr."); return false); OpDescPtr op_desc = node->GetOpDesc(); if (op_desc == nullptr) { return false; @@ -416,7 +423,7 @@ Status AtomicAddrCleanPass::CompileUnknownGraphOp(const vector &atomic_ std::shared_ptr instance = ge::GELib::GetInstance(); if ((instance == nullptr) || !instance->InitFlag()) { REPORT_INNER_ERROR("E19999", "GeLib is not init before, check invalid"); - GELOGE(ge::GE_CLI_GE_NOT_INITIALIZED, "CompileSingleOp failed."); + GELOGE(ge::GE_CLI_GE_NOT_INITIALIZED, "[Check][Param] GeLib is not init before."); return ge::GE_CLI_GE_NOT_INITIALIZED; } @@ -428,10 +435,10 @@ Status AtomicAddrCleanPass::CompileUnknownGraphOp(const vector &atomic_ } string kernel_lib_name = op_desc->GetOpKernelLibName(); if (kernel_lib_name.empty()) { - REPORT_INNER_ERROR("E19999", "Find ops kernel by name:%s failed", - kernel_lib_name.c_str()); - GELOGE(ge::INTERNAL_ERROR, "Get atomic node:%s(%s) kernel lib failed.", atomic_node->GetName().c_str(), - atomic_node->GetType().c_str()); + REPORT_INNER_ERROR("E19999", "Find ops kernel by name of op:%s(%s) failed", + op_desc->GetName().c_str(), op_desc->GetType().c_str()); + GELOGE(ge::INTERNAL_ERROR, "[Get][OpKernelLibName] of op:%s(%s) failed.", op_desc->GetName().c_str(), + op_desc->GetType().c_str()); return ge::INTERNAL_ERROR; } @@ -451,8 +458,8 @@ Status AtomicAddrCleanPass::CompileUnknownGraphOp(const vector &atomic_ GE_TIMESTAMP_ADD(UnknownGraphCompileOp); if (ret != ge::SUCCESS) { REPORT_CALL_ERROR("E19999", "Call CompileOp failed, kernel_lib_name:%s, ret:%d", - kernel_lib_name.c_str(), ret); - GELOGE(ret, "Compile atomic op failed, kernel lib name is %s", kernel_lib_name.c_str()); + kernel_lib_name.c_str(), ret); + GELOGE(ret, "[Compile][AtomicOp] failed, kernel lib name is %s", kernel_lib_name.c_str()); return ret; } } diff --git a/ge/graph/passes/attach_stream_label_pass.cc b/ge/graph/passes/attach_stream_label_pass.cc index d8c81e92..d5b28ec7 100644 --- a/ge/graph/passes/attach_stream_label_pass.cc +++ b/ge/graph/passes/attach_stream_label_pass.cc @@ -29,9 +29,11 @@ Status AttachStreamLabelPass::Run(ComputeGraphPtr graph) { std::map branch_head_nodes; FindNodes(graph, need_label_nodes, enter_nodes, branch_head_nodes); for (const auto &node : need_label_nodes) { - GE_CHK_STATUS_RET(UpdateCondBranch(node, branch_head_nodes), "Update cond branch failed, start node:%s.", node->GetName().c_str()); + GE_CHK_STATUS_RET(UpdateCondBranch(node, branch_head_nodes), "[Update][CondBranch] failed, start node:%s.", + node->GetName().c_str()); } - GE_CHK_STATUS_RET(UpdateEnterNode(enter_nodes), "UpdateEnterNode failed."); + GE_CHK_STATUS_RET(UpdateEnterNode(enter_nodes), + "[Update][EnterNode] in graph:%s failed.", graph->GetName().c_str()); GELOGD("AttachStreamLabelPass Leave."); return SUCCESS; @@ -83,7 +85,7 @@ Status AttachStreamLabelPass::UpdateCondBranch(const NodePtr &node, const std::map &branch_head_nodes) { std::string stream_label; if (AttachFlag(node, stream_label) != SUCCESS) { - GELOGE(FAILED, "Attach flag for node %s failed.", node->GetName().c_str()); + GELOGE(FAILED, "[Attach][Flag] for node %s failed.", node->GetName().c_str()); return FAILED; } @@ -119,9 +121,8 @@ Status AttachStreamLabelPass::UpdateCondBranch(const NodePtr &node, GELOGD("Attach label %s to node: %s.", stream_label.c_str(), tmp_node->GetName().c_str()); auto status = SetStreamLabel(tmp_node, stream_label); if (status != ge::SUCCESS) { - REPORT_CALL_ERROR("E19999", "Set stream_label:%s to op:%s(%s) failed", - stream_label.c_str(), tmp_node->GetName().c_str(), tmp_node->GetType().c_str()); - GELOGE(status, "Set stream label failed."); + GELOGE(status, "[Set][StreamLabel] %s to op:%s(%s) failed.", + stream_label.c_str(), tmp_node->GetName().c_str(), tmp_node->GetType().c_str()); return status; } } @@ -141,7 +142,7 @@ Status AttachStreamLabelPass::AttachFlag(const NodePtr &node, std::string &strea if (node->GetInDataNodes().empty()) { REPORT_INNER_ERROR("E19999", "In data nodes is empty of op:%s(%s), check invalid", node->GetName().c_str(), node->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "node %s has no input_data_node.", node->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Get][InDataNodes] node %s has no input_data_node.", node->GetName().c_str()); return INTERNAL_ERROR; } stream_label = node->GetInDataNodes().at(0)->GetName(); @@ -153,22 +154,21 @@ Status AttachStreamLabelPass::AttachFlag(const NodePtr &node, std::string &strea ATTR_NAME_SWITCH_TRUE_BRANCH_FLAG.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); return FAILED, - "StreamSwitch get attr TRUE_BRANCH_STREAM failed."); + "[Get][Attr] %s of op:%s(%s) failed", ATTR_NAME_SWITCH_TRUE_BRANCH_FLAG.c_str(), + op_desc->GetName().c_str(), op_desc->GetType().c_str()); stream_label += (value ? "_t" : "_f"); auto status = SetActiveLabelList(node, {stream_label}); if (status != ge::SUCCESS) { - REPORT_CALL_ERROR("E19999", "Set active label list:%s to op:%s(%s) failed", - stream_label.c_str(), node->GetName().c_str(), node->GetType().c_str()); - GELOGE(status, "set active_label_list failed."); + GELOGE(status, "[Set][ActiveLabelList] %s to op:%s(%s) failed.", + stream_label.c_str(), node->GetName().c_str(), node->GetType().c_str()); return status; } } else if (type == STREAMMERGE) { stream_label = node->GetName(); auto status = SetStreamLabel(node, stream_label); if (status != ge::SUCCESS) { - REPORT_CALL_ERROR("E19999", "Set stream_label:%s to op:%s(%s) failed", - stream_label.c_str(), node->GetName().c_str(), node->GetType().c_str()); - GELOGE(status, "Set stream label failed."); + GELOGE(status, "[Set][StreamLabel] %s to op:%s(%s) failed.", + stream_label.c_str(), node->GetName().c_str(), node->GetType().c_str()); return status; } } @@ -198,7 +198,7 @@ Status AttachStreamLabelPass::UpdateEnterNode(const std::vector &enter_ for (const auto &pair : enter_active_map) { if (SetEnterLabel(pair.second, pair.first) != SUCCESS) { - GELOGE(FAILED, "Set stream_label for enter_nodes failed."); + GELOGE(FAILED, "[Set][EnterLabel] for enter_nodes failed."); return FAILED; } @@ -211,7 +211,8 @@ Status AttachStreamLabelPass::UpdateEnterNode(const std::vector &enter_ REPORT_CALL_ERROR("E19999", "Get Attr:%s of op:%s(%s) failed", ATTR_NAME_ACTIVE_LABEL_LIST.c_str(), active_node->GetName().c_str(), active_node->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Get attr ATTR_NAME_ACTIVE_LABEL_LIST failed, node: %s.", active_node->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Get][Attr] %s of op:%s(%s) failed.", ATTR_NAME_ACTIVE_LABEL_LIST.c_str(), + active_node->GetName().c_str(), active_node->GetType().c_str()); return INTERNAL_ERROR; } @@ -220,7 +221,7 @@ Status AttachStreamLabelPass::UpdateEnterNode(const std::vector &enter_ nodes.emplace(enter_node); } if (UpdateLoopBranch(nodes, active_label_list[0]) != SUCCESS) { - GELOGE(FAILED, "Update stream_label for loop_branch failed."); + GELOGE(FAILED, "[Update][StreamLabel] %s for loop_branch failed.", active_label_list[0].c_str()); return FAILED; } } @@ -246,9 +247,8 @@ Status AttachStreamLabelPass::SetEnterLabel(const std::vector &enter_no for (const auto &enter_node : enter_nodes) { auto status = SetStreamLabel(enter_node, stream_label); if (status != ge::SUCCESS) { - REPORT_CALL_ERROR("E19999", "Set stream_label:%s to op:%s(%s) failed", - stream_label.c_str(), enter_node->GetName().c_str(), enter_node->GetType().c_str()); - GELOGE(status, "Set stream label failed."); + GELOGE(status, "[Set][StreamLabel] %s to op:%s(%s) failed.", + stream_label.c_str(), enter_node->GetName().c_str(), enter_node->GetType().c_str()); return status; } } @@ -281,9 +281,8 @@ Status AttachStreamLabelPass::UpdateLoopBranch(const std::stack &enter_ GELOGD("Attach label %s to node: %s.", stream_label.c_str(), out_node->GetName().c_str()); auto status = SetStreamLabel(out_node, stream_label); if (status != ge::SUCCESS) { - REPORT_CALL_ERROR("E19999", "Set stream_label:%s to op:%s(%s) failed", - stream_label.c_str(), out_node->GetName().c_str(), out_node->GetType().c_str()); - GELOGE(status, "Set stream label failed."); + GELOGE(status, "[Set][StreamLabel] %s to op:%s(%s) failed.", + stream_label.c_str(), out_node->GetName().c_str(), out_node->GetType().c_str()); return status; } nodes.push(out_node); diff --git a/ge/graph/passes/base_pass.cc b/ge/graph/passes/base_pass.cc index 2f94c6ad..6d933295 100755 --- a/ge/graph/passes/base_pass.cc +++ b/ge/graph/passes/base_pass.cc @@ -142,13 +142,14 @@ void PushToRePassIfSeen(NodePtr &node, const std::pairGetName().c_str()); for (const auto &name_to_pass : names_to_passes) { if (name_to_pass.second == nullptr) { - GELOGE(INTERNAL_ERROR, "There is null pointer in passes(%s), skip it", name_to_pass.first.c_str()); + GELOGE(INTERNAL_ERROR, "[Check][Param] There is null pointer in passes(%s), skip it", name_to_pass.first.c_str()); continue; } @@ -156,8 +157,9 @@ Status RunPasses(NodePtr &node, const NamesToPass &names_to_passes, DuringPassNo name_to_pass.second->init(); auto result = name_to_pass.second->Run(node); if (result != SUCCESS) { - GELOGE(INTERNAL_ERROR, - "Failed to process pass %s on node %s, result " + REPORT_CALL_ERROR("E19999", "process pass %s on node:%s failed, ret:%u", + name_to_pass.first.c_str(), node->GetName().c_str(), result); + GELOGE(INTERNAL_ERROR, "[Process][Pass] %s on node %s failed, result " "%u, the passes will be terminated immediately.", name_to_pass.first.c_str(), node->GetName().c_str(), result); return result; @@ -201,26 +203,31 @@ void ClearOption(NamesToPass names_to_pass) { Status BaseNodePass::IsolateAndDeleteNode(NodePtr &node, const std::vector &io_map) { if (node == nullptr) { - GELOGE(FAILED, "parameter is null."); + REPORT_INNER_ERROR("E19999", "Param node is nullptr, check invalid."); + GELOGE(FAILED, "[Check][Param] parameter node is nullptr."); return FAILED; } GELOGI("Prepare to isolate and delete node, name:%s, type:%s.", node->GetName().c_str(), node->GetType().c_str()); ComputeGraphPtr graph = node->GetOwnerComputeGraph(); if (graph == nullptr) { - GELOGE(FAILED, "[%s] The owner graph must not be null.", node->GetName().c_str()); + REPORT_INNER_ERROR("E19999", "The owner graph of node:%s must not be null.", node->GetName().c_str()); + GELOGE(FAILED, "[Get][OwnerComputeGraph] failed, The owner graph of node:%s must not be null.", + node->GetName().c_str()); return FAILED; } AddRePassNodesWithInOut(node); if (GraphUtils::IsolateNode(node, io_map) != GRAPH_SUCCESS) { - GELOGE(FAILED, "[%s] IsolateNode failed.", node->GetName().c_str()); + REPORT_CALL_ERROR("E19999", "Isolate Node:%s failed", node->GetName().c_str()); + GELOGE(FAILED, "[Isolate][Node] %s failed.", node->GetName().c_str()); return FAILED; } if (GraphUtils::RemoveNodeWithoutRelink(graph, node) != SUCCESS) { - GELOGE(FAILED, "[%s] RemoveNodeWithoutRelink failed.", node->GetName().c_str()); + REPORT_CALL_ERROR("E19999", "call RemoveNodeWithoutRelink for node:%s failed.", node->GetName().c_str()); + GELOGE(FAILED, "[Call][RemoveNodeWithoutRelink] for node:%s failed.", node->GetName().c_str()); return FAILED; } @@ -230,7 +237,8 @@ Status BaseNodePass::IsolateAndDeleteNode(NodePtr &node, const std::vector Status GEPass::Run(const NamesToPass &names_to_passes) { if (graph_ == nullptr) { - GELOGE(INTERNAL_ERROR, "The graph is null"); + REPORT_INNER_ERROR("E19999", "graph_ is nullptr, check invalid."); + GELOGE(INTERNAL_ERROR, "[Check][Param] The graph is nullptr"); return INTERNAL_ERROR; } if (names_to_passes.empty()) { @@ -240,7 +248,7 @@ Status GEPass::Run(const NamesToPass &names_to_passes) { if (depth_ > kMaxRecursiveDepth) { GELOGE(PARAM_INVALID, - "The pass for root graph %s will be terminated because too many nesting" + "[Check][Param] The pass for root graph %s will be terminated because too many nesting" " levels(%d) of subgraphs, last subgraph is %s", root_graph_->GetName().c_str(), depth_, graph_->GetName().c_str()); return PARAM_INVALID; @@ -284,7 +292,7 @@ Status GEPass::RunPassesOneGraph(const NamesToPass &names_to_passes) { auto ret = RunPasses(node, names_to_passes, during_pass_node_set); if (ret != SUCCESS) { - GELOGE(ret, "Failed to process passes on node %s type %s, error code: %u", + GELOGE(ret, "[Process][Passes] on node %s type %s failed, error code:%u", node->GetName().c_str(), node->GetType().c_str(), ret); return ret; } @@ -292,7 +300,7 @@ Status GEPass::RunPassesOneGraph(const NamesToPass &names_to_passes) { bool has_sub_graph = false; ret = RunPassesOnSubGraph(node, names_to_passes, has_sub_graph); if (ret != SUCCESS) { - GELOGE(ret, "Failed to run passes on the sub graph of node %s", node->GetName().c_str()); + GELOGE(ret, "[Run][Passes] on the sub graph of node %s failed", node->GetName().c_str()); return ret; } @@ -301,7 +309,7 @@ Status GEPass::RunPassesOneGraph(const NamesToPass &names_to_passes) { SetFlagOption(kOptimizeAfterSubGraph, names_to_passes); ret = RunPasses(node, names_to_passes, during_pass_node_set); if (ret != SUCCESS) { - GELOGE(ret, "Failed to process passes on node %s type %s, error code: %u", + GELOGE(ret, "[Process][Passes] on node %s type %s failed, error code: %u", node->GetName().c_str(), node->GetType().c_str(), ret); return ret; } @@ -347,7 +355,7 @@ Status GEPass::RunPassesOnSubGraph(const NodePtr &node, const NamesToPass &names GEPass pass(graph, root_graph_, depth_ + 1); auto ret = pass.Run(names_to_passes); if (ret != SUCCESS) { - GELOGE(ret, "Failed to run passes for sub graph %s from node %s", name.c_str(), node->GetName().c_str()); + GELOGE(ret, "[Run][Passes] for sub graph:%s from node:%s failed", name.c_str(), node->GetName().c_str()); return ret; } } diff --git a/ge/graph/passes/bitcast_pass.cc b/ge/graph/passes/bitcast_pass.cc index b5166959..3d3d109c 100644 --- a/ge/graph/passes/bitcast_pass.cc +++ b/ge/graph/passes/bitcast_pass.cc @@ -33,7 +33,7 @@ Status BitcastPass::Run(NodePtr &node) { GELOGD("Bitcast running"); if (node == nullptr) { REPORT_INNER_ERROR("E19999", "Param node is nullptr, check invalid"); - GELOGE(PARAM_INVALID, "Param [node] must not be null."); + GELOGE(PARAM_INVALID, "[Check][Param] Param [node] must not be null."); return PARAM_INVALID; } @@ -44,6 +44,7 @@ Status BitcastPass::Run(NodePtr &node) { OpDescPtr op_desc = node->GetOpDesc(); if (op_desc == nullptr) { REPORT_INNER_ERROR("E19999", "Param op_desc of node is nullptr, check invalid"); + GELOGE(PARAM_INVALID, "[Get][OpDesc] failed, op_desc of node is nullptr"); return PARAM_INVALID; } ge::DataType dst_data_type; @@ -62,22 +63,25 @@ Status BitcastPass::CheckDstDataType(const OpDescPtr op_desc, ge::DataType &dst_ if (!ge::AttrUtils::GetDataType(op_desc, kAttrNameType, dst_data_type)) { REPORT_CALL_ERROR("E19999", "Get Attr:%s of op:%s(%s) failed", - kAttrNameType, op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(PARAM_INVALID, "Node failed to get attribute type."); + kAttrNameType, op_desc->GetName().c_str(), op_desc->GetType().c_str()); + GELOGE(PARAM_INVALID, "[Get][Attr] %s of op:%s(%s) failed", + kAttrNameType, op_desc->GetName().c_str(), op_desc->GetType().c_str()); return PARAM_INVALID; } if (dst_data_type >= ge::DT_UNDEFINED) { REPORT_INNER_ERROR("E19999", "Param dst_data_type:%d check invalid, op:%s(%s)", dst_data_type, op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(PARAM_INVALID, "dst_data_type[%s] is not valid.", - TypeUtils::DataTypeToSerialString(dst_data_type).c_str()); + GELOGE(PARAM_INVALID, "[Check][Param] dst_data_type[%s] is not valid, op:%s(%s)", + TypeUtils::DataTypeToSerialString(dst_data_type).c_str(), + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return PARAM_INVALID; } if (op_desc->GetOutputDescPtr(0) == nullptr) { REPORT_INNER_ERROR("E19999", "Index 0 ouput desc of op:%s(%s) not exist, check invalid", op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(PARAM_INVALID, "Bitcast node outputDesc is null."); + GELOGE(PARAM_INVALID, "[Get][OutputDescPtr] Index 0 ouput desc of op:%s(%s) not exist.", + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return PARAM_INVALID; } if (op_desc->GetOutputDescPtr(0)->GetDataType() != dst_data_type) { @@ -85,9 +89,10 @@ Status BitcastPass::CheckDstDataType(const OpDescPtr op_desc, ge::DataType &dst_ "check invalid", op_desc->GetName().c_str(), op_desc->GetType().c_str(), TypeUtils::DataTypeToSerialString(dst_data_type).c_str(), TypeUtils::DataTypeToSerialString(op_desc->GetOutputDescPtr(0)->GetDataType()).c_str()); - GELOGE(PARAM_INVALID, "dst_data_type[%s] is not equal to output_data_type[%s].", + GELOGE(PARAM_INVALID, "[Check][Param] dst_data_type[%s] is not equal to output_data_type[%s], op:%s(%s).", TypeUtils::DataTypeToSerialString(dst_data_type).c_str(), - TypeUtils::DataTypeToSerialString(op_desc->GetOutputDescPtr(0)->GetDataType()).c_str()); + TypeUtils::DataTypeToSerialString(op_desc->GetOutputDescPtr(0)->GetDataType()).c_str(), + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return PARAM_INVALID; } return SUCCESS; @@ -99,7 +104,8 @@ Status BitcastPass::CheckOutputShape(const OpDescPtr op_desc, const ge::DataType if (input_tensor_desc == nullptr) { REPORT_INNER_ERROR("E19999", "Index 0 input desc of op:%s(%s) not exist, check invalid", op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(PARAM_INVALID, "input_tensor_desc must not be null."); + GELOGE(PARAM_INVALID, "[Check][Param] input_tensor_desc(index:0) of op:%s(%s) must not be null.", + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return PARAM_INVALID; } @@ -109,8 +115,9 @@ Status BitcastPass::CheckOutputShape(const OpDescPtr op_desc, const ge::DataType REPORT_INNER_ERROR("E19999", "ori_data_type:%d of index 0 input desc in op:%s(%s), " "check invalid", ori_data_type, op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(PARAM_INVALID, "ori_data_type[%s] is not valid.", - TypeUtils::DataTypeToSerialString(ori_data_type).c_str()); + GELOGE(PARAM_INVALID, "[Check][Param] ori_data_type[%s] of input desc(index 0) in op:%s(%s) is not valid.", + TypeUtils::DataTypeToSerialString(ori_data_type).c_str(), + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return PARAM_INVALID; } @@ -121,7 +128,7 @@ Status BitcastPass::CheckOutputShape(const OpDescPtr op_desc, const ge::DataType BitcastPass::kVecInt64 dim_vec(input_tensor_desc->GetShape().GetDims()); if (CalcAndUpdateShape(dim_vec, ori_data_type, dst_data_type) != SUCCESS) { - GELOGE(PARAM_INVALID, "CalcAndUpdateShape failed."); + GELOGE(PARAM_INVALID, "[Call][CalcAndUpdateShape] failed."); return PARAM_INVALID; } @@ -130,7 +137,9 @@ Status BitcastPass::CheckOutputShape(const OpDescPtr op_desc, const ge::DataType "check invalid", formats::JoinToString(output_tensor_desc->GetShape().GetDims()).c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str(), formats::JoinToString(dim_vec).c_str()); - GELOGE(PARAM_INVALID, "out_put_shape is different from expectations."); + GELOGE(PARAM_INVALID, "[Check][Param] Shape:%s of index 0 output desc in op:%s(%s), different from expect shape:%s", + formats::JoinToString(output_tensor_desc->GetShape().GetDims()).c_str(), + op_desc->GetName().c_str(), op_desc->GetType().c_str(), formats::JoinToString(dim_vec).c_str()); return PARAM_INVALID; } @@ -141,7 +150,7 @@ Status BitcastPass::CalcAndUpdateShape(BitcastPass::kVecInt64 &dim_vec, ge::Data ge::DataType dst_data_type) { if (dim_vec.size() == 0) { REPORT_INNER_ERROR("E19999", "Param dim_vec is empty, check invalid"); - GELOGE(PARAM_INVALID, "Pre node shape size is zero."); + GELOGE(PARAM_INVALID, "[Check][Param] Param dim_vec is empty."); return PARAM_INVALID; } int64_t ori_data_size = GetSizeByDataType(ori_data_type); @@ -155,7 +164,10 @@ Status BitcastPass::CalcAndUpdateShape(BitcastPass::kVecInt64 &dim_vec, ge::Data "check invalid", ori_data_size, TypeUtils::DataTypeToSerialString(ori_data_type).c_str(), dst_data_size, TypeUtils::DataTypeToSerialString(dst_data_type).c_str()); - GELOGE(PARAM_INVALID, "ori_data_size is not divisible by dst_data_size."); + GELOGE(PARAM_INVALID, + "[Check][Param] size:%ld of ori_data_type:%s is not divisible by size:%ld of dst_data_type:%s.", + ori_data_size, TypeUtils::DataTypeToSerialString(ori_data_type).c_str(), + dst_data_size, TypeUtils::DataTypeToSerialString(dst_data_type).c_str()); return PARAM_INVALID; } dim_vec.push_back(ori_data_size / dst_data_size); @@ -166,7 +178,10 @@ Status BitcastPass::CalcAndUpdateShape(BitcastPass::kVecInt64 &dim_vec, ge::Data "check invalid", dst_data_size, TypeUtils::DataTypeToSerialString(dst_data_type).c_str(), ori_data_size, TypeUtils::DataTypeToSerialString(ori_data_type).c_str()); - GELOGE(PARAM_INVALID, "dst_data_size is not divisible by ori_data_size."); + GELOGE(PARAM_INVALID, + "[Check][Param] size:%ld of dst_data_type:%s is not divisible by size:%ld of ori_data_type:%s.", + dst_data_size, TypeUtils::DataTypeToSerialString(dst_data_type).c_str(), + ori_data_size, TypeUtils::DataTypeToSerialString(ori_data_type).c_str()); return PARAM_INVALID; } @@ -174,7 +189,9 @@ Status BitcastPass::CalcAndUpdateShape(BitcastPass::kVecInt64 &dim_vec, ge::Data REPORT_INNER_ERROR("E19999", "The last dim:%ld in param dim_vec is not equal to " "dst_data_size:%ld / ori_data_size:%ld, check invalid", dim_vec[dim_vec.size() - 1], dst_data_size, ori_data_size); - GELOGE(PARAM_INVALID, "The last dim is not equal to dst_data_size / ori_data_size."); + GELOGE(PARAM_INVALID, "[Check][Param] The last dim:%ld in param dim_vec is not equal to " + "dst_data_size:%ld / ori_data_size:%ld", + dim_vec[dim_vec.size() - 1], dst_data_size, ori_data_size); return PARAM_INVALID; } dim_vec.pop_back(); diff --git a/ge/graph/passes/cast_remove_pass.cc b/ge/graph/passes/cast_remove_pass.cc index 7e2bb7bb..564b311d 100644 --- a/ge/graph/passes/cast_remove_pass.cc +++ b/ge/graph/passes/cast_remove_pass.cc @@ -26,13 +26,13 @@ namespace ge { Status CastRemovePass::Run(NodePtr &node) { if (node == nullptr) { REPORT_INNER_ERROR("E19999", "Param node is nullptr, check invalid"); - GELOGE(PARAM_INVALID, "Param [node] must not be null."); + GELOGE(PARAM_INVALID, "[Check][Param] Param [node] must not be null."); return PARAM_INVALID; } OpDescPtr op_desc = node->GetOpDesc(); if (op_desc == nullptr) { REPORT_INNER_ERROR("E19999", "Param op_desc of node is nullptr, check invalid"); - GELOGE(PARAM_INVALID, "OpDesc of param [node] must not be null."); + GELOGE(PARAM_INVALID, "[Get][OpDesc] OpDesc of param [node] must not be null."); return PARAM_INVALID; } @@ -49,7 +49,7 @@ Status CastRemovePass::Run(NodePtr &node) { OpDescPtr end_op_desc = end_node->GetOpDesc(); if (end_op_desc == nullptr) { REPORT_INNER_ERROR("E19999", "op_desc of end_node is nullptr, check invalid"); - GELOGE(PARAM_INVALID, "OpDesc of end node must not be null."); + GELOGE(PARAM_INVALID, "[Get][OpDesc] OpDesc of end node must not be null."); return PARAM_INVALID; } @@ -104,7 +104,7 @@ Status CastRemovePass::RemoveCast(DataType &type, std::vector &nodes_to if (IsolateAndDeleteNode(node, {0}) != SUCCESS) { REPORT_CALL_ERROR("E19999", "Isolate and delete node:%s(%s) failed", node->GetName().c_str(), node->GetType().c_str()); - GELOGE(FAILED, "IsolateAndDeleteNode %s failed.", node->GetName().c_str()); + GELOGE(FAILED, "[IsolateAndDelete][Node] %s failed.", node->GetName().c_str()); return FAILED; } } @@ -120,7 +120,7 @@ Status CastRemovePass::RemoveCast(DataType &type, std::vector &nodes_to OpDescPtr op_desc = node->GetOpDesc(); if (op_desc == nullptr) { REPORT_INNER_ERROR("E19999", "Find nullptr op_desc in node, check invalid"); - GELOGE(FAILED, "OpDesc must not be null."); + GELOGE(FAILED, "[Get][OpDesc] OpDesc must not be null."); return FAILED; } @@ -132,7 +132,8 @@ Status CastRemovePass::RemoveCast(DataType &type, std::vector &nodes_to REPORT_CALL_ERROR("E19999", "Set Attr:%s of op:%s(%s) failed", ATTR_NEED_COMPILE.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(FAILED, "Set ATTR_NEED_COMPILE Attr fail."); + GELOGE(FAILED, "[Set][Attr] %s of op:%s(%s) failed", ATTR_NEED_COMPILE.c_str(), + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return FAILED; } auto in_desc = op_desc->MutableInputDesc(0); diff --git a/ge/graph/passes/cast_translate_pass.cc b/ge/graph/passes/cast_translate_pass.cc index 37e9bc83..d49424c8 100644 --- a/ge/graph/passes/cast_translate_pass.cc +++ b/ge/graph/passes/cast_translate_pass.cc @@ -33,7 +33,7 @@ namespace ge { bool CastTranslatePass::CheckInAndOutDataAnchor(NodePtr &node) const { if (node == nullptr) { - GELOGE(FAILED, "parameter is null."); + GELOGE(FAILED, "[Check][Param] parameter node is nullptr."); return false; } if (node->GetOpDesc() == nullptr) { @@ -191,7 +191,7 @@ bool CastTranslatePass::IsOpSupportedOptimize(NodePtr &cast_node, NodePtr &trans } bool CastTranslatePass::CheckOpSupportOptimize(NodePtr &node, bool &is_src_cast) { - GE_IF_BOOL_EXEC(node == nullptr, GELOGE(FAILED, "node is null."); return false); + GE_IF_BOOL_EXEC(node == nullptr, GELOGE(FAILED, "[Check][Param] node is nullptr."); return false); auto out_node = node->GetOutDataNodes().at(0); // N dst nodes have the same datatype and format, check the first node if (is_src_cast) { @@ -249,14 +249,19 @@ Status CastTranslatePass::FuseDstNTranslates(NodePtr &node) { GE_CHECK_NOTNULL(out_data_node); AddRePassNodesWithInOut(out_data_node); // Has checked nodes only has one in data anchor one out data anchor - GE_CHK_GRAPH_STATUS_RET(NodeUtils::MoveOutputEdges(out_data_node, base_node), "move out put edge failed"); + GE_CHK_GRAPH_STATUS_RET(NodeUtils::MoveOutputEdges(out_data_node, base_node), + "[Move][OutputEdges] failed, out data node:%s, index:0", + base_node->GetName().c_str()); // Relink in control anchor, delete in data anchor auto in_ctr_anchor = out_data_node->GetInControlAnchor(); GE_CHECK_NOTNULL(in_ctr_anchor); for (const auto &peer_anchor : in_ctr_anchor->GetPeerOutControlAnchors()) { GE_CHECK_NOTNULL(base_node->GetInControlAnchor()); - GE_CHK_GRAPH_STATUS_RET(base_node->GetInControlAnchor()->LinkFrom(peer_anchor), "link from peer anchor failed"); + GE_CHK_GRAPH_STATUS_RET(base_node->GetInControlAnchor()->LinkFrom(peer_anchor), + "[Add][Edge] between %s and %s failed", + base_node->GetInControlAnchor()->GetOwnerNode()->GetName().c_str(), + peer_anchor->GetOwnerNode()->GetName().c_str()); } in_ctr_anchor->UnlinkAll(); out_data_node->GetAllInDataAnchors().at(0)->UnlinkAll(); @@ -266,7 +271,8 @@ Status CastTranslatePass::FuseDstNTranslates(NodePtr &node) { if (GraphUtils::RemoveNodeWithoutRelink(graph, out_data_node) != SUCCESS) { REPORT_CALL_ERROR("E19999", "Remove node:%s(%s) without relink in graph:%s failed", out_data_node->GetName().c_str(), out_data_node->GetType().c_str(), graph->GetName().c_str()); - GELOGE(FAILED, "[%s] RemoveNodeWithoutRelink failed.", out_data_node->GetName().c_str()); + GELOGE(FAILED, "[Remove][Node] %s(%s) without relink in graph:%s failed", + out_data_node->GetName().c_str(), out_data_node->GetType().c_str(), graph->GetName().c_str()); return FAILED; } AddNodeDeleted(out_data_node); diff --git a/ge/graph/passes/common_subexpression_elimination_pass.cc b/ge/graph/passes/common_subexpression_elimination_pass.cc index a95d0077..852ed98a 100644 --- a/ge/graph/passes/common_subexpression_elimination_pass.cc +++ b/ge/graph/passes/common_subexpression_elimination_pass.cc @@ -109,8 +109,8 @@ Status CommonSubexpressionEliminationPass::Run(ComputeGraphPtr graph) { REPORT_CALL_ERROR("E19999", "Replace node:%s(%s)'s anchor by node:%s(%s) failed", node->GetName().c_str(), node->GetType().c_str(), iter->second->GetName().c_str(), iter->second->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to replace node %s by node %s error node %u", - node->GetName().c_str(), iter->second->GetName().c_str(), ret); + GELOGE(INTERNAL_ERROR, "[Replace][Node] %s by node %s failed, ret:%u", + node->GetName().c_str(), iter->second->GetName().c_str(), ret); return INTERNAL_ERROR; } @@ -120,7 +120,8 @@ Status CommonSubexpressionEliminationPass::Run(ComputeGraphPtr graph) { if (ret != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Remove node:%s(%s) without relink in graph:%s failed", node->GetName().c_str(), node->GetType().c_str(), graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to remove node %s from graph", node->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Remove][Node] %s from graph:%s failed", node->GetName().c_str(), + graph->GetName().c_str()); return INTERNAL_ERROR; } diff --git a/ge/graph/passes/compile_nodes_pass.cc b/ge/graph/passes/compile_nodes_pass.cc index 76330dc6..d0dcec16 100755 --- a/ge/graph/passes/compile_nodes_pass.cc +++ b/ge/graph/passes/compile_nodes_pass.cc @@ -42,7 +42,7 @@ graphStatus CompileNodesPass::Run(ComputeGraphPtr graph) { std::shared_ptr instance = ge::GELib::GetInstance(); if (instance == nullptr || !instance->InitFlag()) { REPORT_INNER_ERROR("E19999", "Gelib not init before, check invalid"); - GELOGE(ge::GE_CLI_GE_NOT_INITIALIZED, "Run CompileNodesPass failed."); + GELOGE(ge::GE_CLI_GE_NOT_INITIALIZED, "[Check][Param] Gelib not init before."); return ge::GE_CLI_GE_NOT_INITIALIZED; } std::unordered_map> kernel_to_compile_nodes; @@ -71,7 +71,7 @@ graphStatus CompileNodesPass::Run(ComputeGraphPtr graph) { kernel_to_compile_nodes.insert(std::make_pair(kernel_lib_name, node_vec)); } } else { - GELOGE(GRAPH_FAILED, "Get node:%s, type:%s supported kernel failed.", node->GetName().c_str(), + GELOGE(GRAPH_FAILED, "[Get][SupportedKernel] for node:%s(%s) failed.", node->GetName().c_str(), node->GetType().c_str()); return GRAPH_FAILED; } @@ -79,7 +79,7 @@ graphStatus CompileNodesPass::Run(ComputeGraphPtr graph) { // compile node follow different kernel, currently only TBE kernel auto result = CompileNodes(instance, kernel_to_compile_nodes); if (result != GRAPH_SUCCESS) { - GELOGE(result, "Compile op failed."); + GELOGE(result, "[Compile][Op] failed, ret:%u.", result); return result; } GELOGD("[CompileNodesPass]: Optimize success."); @@ -91,7 +91,7 @@ graphStatus CompileNodesPass::GetSupportedKernel(const NodePtr &node, const std: string &kernel_lib_name) { auto op_desc = node->GetOpDesc(); if (op_desc == nullptr) { - GELOGE(ge::GE_GRAPH_PARAM_NULLPTR, "Get op %s opdesc failed", node->GetName().c_str()); + GELOGE(ge::GE_GRAPH_PARAM_NULLPTR, "[Get][OpDesc] failed, op of param node is nullptr."); return ge::GE_GRAPH_PARAM_NULLPTR; } // reset op kernel lib, find supported kernel @@ -102,7 +102,7 @@ graphStatus CompileNodesPass::GetSupportedKernel(const NodePtr &node, const std: if (kernel_lib_name.empty()) { REPORT_INNER_ERROR("E19999", "kernel_lib_name in op:%s(%s) is empty, check invalid", op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(GRAPH_FAILED, "Get node:%s, type:%s kernel lib failed.", node->GetName().c_str(), + GELOGE(GRAPH_FAILED, "[Get][OpKernelLib] for node:%s(%s) failed.", node->GetName().c_str(), op_desc->GetType().c_str()); return GRAPH_FAILED; } @@ -111,7 +111,7 @@ graphStatus CompileNodesPass::GetSupportedKernel(const NodePtr &node, const std: if (kernel_info == nullptr) { REPORT_INNER_ERROR("E19999", "Find ops kernel by name:%s failed for op:%s(%s)", kernel_lib_name.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(ge::GE_GRAPH_PARAM_NULLPTR, "Get op %s ops kernel info store failed", node->GetName().c_str()); + GELOGE(ge::GE_GRAPH_PARAM_NULLPTR, "[Get][OpsKernelInfoStore] for op:%s failed", node->GetName().c_str()); return ge::GE_GRAPH_PARAM_NULLPTR; } @@ -144,13 +144,13 @@ graphStatus CompileNodesPass::GetSupportedKernel(const NodePtr &node, const std: REPORT_INPUT_ERROR("E13002", std::vector({"optype", "opskernel", "reason"}), std::vector({op_desc->GetType(), it.first, it.second})); GELOGE(GE_GRAPH_ASSIGN_ENGINE_FAILED, - "CheckAccuracySupport:Op type %s of ops kernel %s is unsupported, reason:%s", + "[Call][CheckAccuracySupport] for Op type %s of ops kernel %s is unsupported, reason:%s", op_desc->GetType().c_str(), it.first.c_str(), it.second.c_str()); } REPORT_INPUT_ERROR("E13003", std::vector({"opname", "optype"}), std::vector({op_desc->GetName(), op_desc->GetType()})); - GELOGE(GRAPH_FAILED, "Cannot find kernel lib support node:%s, type:%s , get kernel lib failed.", + GELOGE(GRAPH_FAILED, "[Check][Param] Cannot find kernel lib support node:%s, type:%s , get kernel lib failed.", node->GetName().c_str(), op_desc->GetType().c_str()); return GRAPH_FAILED; } @@ -173,9 +173,8 @@ graphStatus CompileNodesPass::CompileNodes(const std::shared_ptr instance for (auto &kernel_nodes : kernel_to_compile_nodes) { kernel_info = instance->OpsKernelManagerObj().GetOpsKernelInfoStore(kernel_nodes.first); if (kernel_info == nullptr) { - REPORT_INNER_ERROR("E19999", "Find ops kernel by name:%s failed", - kernel_nodes.first.c_str()); - GELOGE(ge::GE_GRAPH_PARAM_NULLPTR, "Get op %s ops kernel info store failed", kernel_nodes.first.c_str()); + REPORT_INNER_ERROR("E19999", "Find ops kernel by name:%s failed", kernel_nodes.first.c_str()); + GELOGE(ge::GE_GRAPH_PARAM_NULLPTR, "[Get][OpsKernelInfoStore] for op %s failed", kernel_nodes.first.c_str()); return ge::GE_GRAPH_PARAM_NULLPTR; } string reason; @@ -192,7 +191,7 @@ graphStatus CompileNodesPass::CompileNodes(const std::shared_ptr instance if (ret != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Call CompileOp failed, kernel_lib_name:%s, ret:%d", kernel_nodes.first.c_str(), ret); - GELOGE(ret, "Compile op failed, kernel name is %s", kernel_nodes.first.c_str()); + GELOGE(ret, "[Compile][Op] failed, kernel name is %s", kernel_nodes.first.c_str()); return GRAPH_FAILED; } } diff --git a/ge/graph/passes/cond_pass.cc b/ge/graph/passes/cond_pass.cc index c274df49..116e4f89 100644 --- a/ge/graph/passes/cond_pass.cc +++ b/ge/graph/passes/cond_pass.cc @@ -32,7 +32,7 @@ Status CondPass::Run(NodePtr &node) { if (ret == NOT_CHANGED) { return SUCCESS; } else if (ret != SUCCESS) { - GELOGE(FAILED, "Get cond_info for node %s failed.", node->GetName().c_str()); + GELOGE(FAILED, "[Get][CondInfo] for node %s failed.", node->GetName().c_str()); return FAILED; } @@ -48,19 +48,19 @@ Status CondPass::Run(NodePtr &node) { if (cond_tensor.MutableShape().GetDim(0) == UNKNOWN_DIM_NUM) { GELOGI("Output tensor rank of Cond is unknown."); if (cond_tensor.GetDataType() == DT_STRING) { - GE_CHK_STATUS_RET(HandleStringCond(graph, peer_out_anchor, cond_in_anchor), "HandleStringCond for %s failed.", - op_desc->GetName().c_str()) + GE_CHK_STATUS_RET(HandleStringCond(graph, peer_out_anchor, cond_in_anchor), + "[Handle][StringCond] for op:%s failed.", op_desc->GetName().c_str()) } return SUCCESS; } if (!cond_tensor.GetShape().IsScalar()) { - GE_CHK_STATUS_RET(HandleNonScalarCond(graph, peer_out_anchor, cond_in_anchor), "HandleNonScalarCond for %s failed.", - op_desc->GetName().c_str()) + GE_CHK_STATUS_RET(HandleNonScalarCond(graph, peer_out_anchor, cond_in_anchor), + "[Handle][NonScalarCond] for op:%s failed.", op_desc->GetName().c_str()) } else { switch (cond_tensor.GetDataType()) { case DT_STRING: - GE_CHK_STATUS_RET(HandleStringCond(graph, peer_out_anchor, cond_in_anchor), "HandleStringCond for %s failed.", - op_desc->GetName().c_str()) + GE_CHK_STATUS_RET(HandleStringCond(graph, peer_out_anchor, cond_in_anchor), + "[Handle][StringCond] for op:%s failed.", op_desc->GetName().c_str()) break; case DT_BOOL: case DT_FLOAT: @@ -70,7 +70,7 @@ Status CondPass::Run(NodePtr &node) { case DT_INT8: case DT_INT64: GE_CHK_STATUS_RET(HandleScalarCond(graph, peer_out_anchor, cond_in_anchor, cond_tensor.GetDataType()), - "HandleScalarCond for %s failed.", op_desc->GetName().c_str()) + "[Handle][ScalarCond] for op:%s failed.", op_desc->GetName().c_str()) break; case DT_INT32: break; @@ -79,7 +79,9 @@ Status CondPass::Run(NodePtr &node) { "data_type:%d of index:%d input tensor in op:%s(%s) check invalid", cond_tensor.GetDataType(), cond_in_anchor->GetIdx(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(FAILED, "UpdateInputDesc for node %s failed.", op_desc->GetName().c_str()); + GELOGE(FAILED, "[Check][Param] data_type:%d of index:%d input tensor in op:%s(%s) is invalid", + cond_tensor.GetDataType(), cond_in_anchor->GetIdx(), + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return FAILED; } } @@ -91,7 +93,8 @@ Status CondPass::Run(NodePtr &node) { if (op_desc->UpdateInputDesc(cond_in_anchor->GetIdx(), cond_tensor) != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Update input desc of op:%s(%s) failed, index:%d", op_desc->GetName().c_str(), op_desc->GetType().c_str(), cond_in_anchor->GetIdx()); - GELOGE(FAILED, "UpdateInputDesc for node %s failed.", op_desc->GetName().c_str()); + GELOGE(FAILED, "[Update][InputDesc] for op:%s(%s) failed, index:%d", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), cond_in_anchor->GetIdx()); return FAILED; } @@ -112,12 +115,12 @@ Status CondPass::GetCondInfo(const NodePtr &node, ComputeGraphPtr &graph, OutDat std::string type = node->GetType(); if (kIfOpTypes.count(type) != 0) { if (GetCondInfoForIf(node, graph, peer_out_anchor, cond_in_anchor) != SUCCESS) { - GELOGE(FAILED, "Get cond_info for if node failed."); + GELOGE(FAILED, "[Get][CondInfo] for if node:%s failed.", node->GetName().c_str()); return FAILED; } } else if (kWhileOpTypes.count(type) != 0) { if (GetCondInfoForWhile(node, graph, peer_out_anchor, cond_in_anchor) != SUCCESS) { - GELOGE(FAILED, "Get cond_info for while node failed."); + GELOGE(FAILED, "[Get][CondInfo] for while node:%s failed.", node->GetName().c_str()); return FAILED; } } else { @@ -166,8 +169,9 @@ Status CondPass::GetCondInfoForWhile(const NodePtr &node, ComputeGraphPtr &graph if (iter == subgraph_names_to_index.end()) { REPORT_INNER_ERROR("E19999", "subgraph name:%s not exist in SubgraphNameIndexes map of op:%s(%s), " "check invalid", ATTR_NAME_WHILE_COND.c_str(), - op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(FAILED, "Get cond_graph index failed, while_node:%s.", node->GetName().c_str()); + op_desc->GetName().c_str(), op_desc->GetType().c_str()); + GELOGE(FAILED, "subgraph name:%s not exist in SubgraphNameIndexes map of op:%s(%s)", ATTR_NAME_WHILE_COND.c_str(), + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return FAILED; } std::string cond_graph_instance_name = op_desc->GetSubgraphInstanceName(iter->second); @@ -181,7 +185,7 @@ Status CondPass::GetCondInfoForWhile(const NodePtr &node, ComputeGraphPtr &graph if (output_num != 1) { REPORT_INNER_ERROR("E19999", "Input data anchor num:%u of op:%s(%s) not equal to 1, check invalid", output_num, op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(FAILED, "output size of cond_graph is invalid, expect 1 but %u exactly, while_node:%s.", + GELOGE(FAILED, "[Check][Param] output size of cond_graph is invalid, expect 1 but %u exactly, while_node:%s.", output_num, node->GetName().c_str()); return FAILED; } @@ -239,7 +243,7 @@ Status CondPass::HandleScalarCond(const ComputeGraphPtr &graph, const OutDataAnc std::string cast_name = cond_in_anchor->GetOwnerNode()->GetName() + "_Cast"; NodePtr cast_node = AddCastNode(graph, cast_name, tensor, src_type, DT_INT32); if (cast_node == nullptr) { - GELOGE(FAILED, "Add Cast node failed, name:%s.", cast_name.c_str()); + GELOGE(FAILED, "[Add][CastNode] failed, name:%s.", cast_name.c_str()); return FAILED; } @@ -250,7 +254,7 @@ Status CondPass::HandleScalarCond(const ComputeGraphPtr &graph, const OutDataAnc peer_out_anchor->GetOwnerNode()->GetType().c_str(), cond_in_anchor->GetOwnerNode()->GetName().c_str(), cond_in_anchor->GetOwnerNode()->GetType().c_str()); - GELOGE(FAILED, "Insert Cast node %s between %s->%s failed.", + GELOGE(FAILED, "[Insert][CastNode] %s between %s->%s failed.", cast_node->GetName().c_str(), peer_out_anchor->GetOwnerNode()->GetName().c_str(), cond_in_anchor->GetOwnerNode()->GetName().c_str()); return FAILED; @@ -287,14 +291,16 @@ Status CondPass::InsertNode(const ComputeGraphPtr &graph, const OutDataAnchorPtr if (op_desc == nullptr) { REPORT_CALL_ERROR("E19999", "Create op_desc:%s(%s) failed", (in_data_anchor->GetOwnerNode()->GetName() + "_" + type).c_str(), type.c_str()); - GELOGE(FAILED, "Create op_desc failed."); + GELOGE(FAILED, "[Create][OpDesc] %s(%s) failed.", + (in_data_anchor->GetOwnerNode()->GetName() + "_" + type).c_str(), type.c_str()); return FAILED; } NodePtr new_node = graph->AddNode(op_desc); if (new_node == nullptr) { REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s failed", op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str()); - GELOGE(FAILED, "Create %s node failed.", type.c_str()); + GELOGE(FAILED, "[Add][Node] %s(%s) to graph:%s failed", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str()); return FAILED; } AddRePassNode(new_node); @@ -306,9 +312,10 @@ Status CondPass::InsertNode(const ComputeGraphPtr &graph, const OutDataAnchorPtr peer_out_anchor->GetOwnerNode()->GetType().c_str(), in_data_anchor->GetOwnerNode()->GetName().c_str(), in_data_anchor->GetOwnerNode()->GetType().c_str()); - GELOGE(FAILED, "Insert %s node %s between %s->%s failed.", type.c_str(), - new_node->GetName().c_str(), peer_out_anchor->GetOwnerNode()->GetName().c_str(), - in_data_anchor->GetOwnerNode()->GetName().c_str()); + GELOGE(FAILED, "[Insert][Node] %s(%s) between %s(%s)->%s(%s) failed", + new_node->GetName().c_str(), new_node->GetType().c_str(), + peer_out_anchor->GetOwnerNode()->GetName().c_str(), peer_out_anchor->GetOwnerNode()->GetType().c_str(), + in_data_anchor->GetOwnerNode()->GetName().c_str(), in_data_anchor->GetOwnerNode()->GetType().c_str()); return FAILED; } @@ -337,20 +344,22 @@ NodePtr CondPass::AddCastNode(const ComputeGraphPtr &graph, const std::string &n OpDescBuilder op_desc_builder(name, CAST); OpDescPtr cast_desc = op_desc_builder.AddInput("x", in_tensor).AddOutput("y", out_tensor).Build(); if (cast_desc == nullptr) { - REPORT_CALL_ERROR("E19999", "Create op_desc:%s(%s) failed", - name.c_str(), CAST); - GELOGE(FAILED, "Create cast op_desc failed, name: %s.", name.c_str()); + REPORT_CALL_ERROR("E19999", "Create op_desc:%s(%s) failed", name.c_str(), CAST); + GELOGE(FAILED, "[Create][OpDesc] failed, name:%s(%s).", name.c_str(), CAST); return nullptr; } if (!(AttrUtils::SetInt(cast_desc, CAST_ATTR_SRCT, src) && AttrUtils::SetInt(cast_desc, CAST_ATTR_DSTT, dst) && AttrUtils::SetInt(cast_desc, CAST_ATTR_DST_TYPE, dst) && AttrUtils::SetBool(cast_desc, CAST_ATTR_TRUNCATE, false))) { - REPORT_CALL_ERROR("E19999", "Set Attr:%s,%s,%s,%s to node:%s(%s) not all success", + REPORT_CALL_ERROR("E19999", "Set Attr:%s, %s, %s, %s to node:%s(%s) not all success", CAST_ATTR_SRCT.c_str(), CAST_ATTR_DSTT.c_str(), CAST_ATTR_DST_TYPE.c_str(), CAST_ATTR_TRUNCATE.c_str(), cast_desc->GetName().c_str(), cast_desc->GetType().c_str()); - GELOGE(FAILED, "Set CAST_ATTR failed, node: %s.", name.c_str()); + GELOGE(FAILED, "[Set][Attr] %s, %s, %s, %s to node:%s(%s) not all success", + CAST_ATTR_SRCT.c_str(), CAST_ATTR_DSTT.c_str(), + CAST_ATTR_DST_TYPE.c_str(), CAST_ATTR_TRUNCATE.c_str(), + cast_desc->GetName().c_str(), cast_desc->GetType().c_str()); return nullptr; } @@ -358,7 +367,8 @@ NodePtr CondPass::AddCastNode(const ComputeGraphPtr &graph, const std::string &n if (cast_node == nullptr) { REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s failed", cast_desc->GetName().c_str(), cast_desc->GetType().c_str(), graph->GetName().c_str()); - GELOGE(FAILED, "Add cast node failed, name: %s.", name.c_str()); + GELOGE(FAILED, "[Add][Node] %s(%s) to graph:%s failed", + cast_desc->GetName().c_str(), cast_desc->GetType().c_str(), graph->GetName().c_str()); return nullptr; } AddRePassNode(cast_node); diff --git a/ge/graph/passes/cond_remove_pass.cc b/ge/graph/passes/cond_remove_pass.cc index 74568c2f..478858a9 100644 --- a/ge/graph/passes/cond_remove_pass.cc +++ b/ge/graph/passes/cond_remove_pass.cc @@ -41,7 +41,7 @@ Status CondRemovePass::Run(NodePtr &node) { if (ret == NOT_CHANGED) { return SUCCESS; } else if (ret != SUCCESS) { - GELOGE(FAILED, "Get cond_info for node %s failed.", node->GetName().c_str()); + GELOGE(FAILED, "[Get][CondInfo] for node %s failed.", node->GetName().c_str()); return FAILED; } int32_t cond_index = 0; @@ -90,7 +90,7 @@ Status CondRemovePass::RemoveDeadCondLink(const int32_t index, const NodePtr &no peerout_anchor->GetOwnerNode()->GetType().c_str(), peerout_anchor->GetIdx(), in_anchor->GetOwnerNode()->GetName().c_str(), in_anchor->GetOwnerNode()->GetType().c_str(), in_anchor->GetIdx()); - GELOGE(FAILED, "Remove edge from node %s index %d to node %s index %d.", + GELOGE(FAILED, "[Remove][Edge] from node %s index %d to node %s index %d.", peerout_anchor->GetOwnerNode()->GetName().c_str(), peerout_anchor->GetIdx(), in_anchor->GetOwnerNode()->GetName().c_str(), in_anchor->GetIdx()); return FAILED; @@ -105,7 +105,7 @@ Status CondRemovePass::GetCaseChosenBranch(const NodePtr &node, const uint32_t c if (subgraph_names_size == 0) { REPORT_INNER_ERROR("E19999", "subgraph size of op:%s(%s) is 0, check invavlid", node->GetName().c_str(), node->GetType().c_str()); - GELOGE(FAILED, "Node %s has none subgraph.", node->GetName().c_str()); + GELOGE(FAILED, "[Check][Param] Node %s has none subgraph.", node->GetName().c_str()); return ge::FAILED; } // If cond index is over the maimum subgraph number, choose the last subgraph @@ -116,7 +116,7 @@ Status CondRemovePass::GetCaseChosenBranch(const NodePtr &node, const uint32_t c if (chosen_branch_name.empty()) { REPORT_INNER_ERROR("E19999", "Get subgraph name from op:%s(%s) by index:%u failed", node->GetName().c_str(), node->GetType().c_str(), cond_index_new); - GELOGE(FAILED, "Node %s has no subgraph, index is %u.", node->GetName().c_str(), cond_index_new); + GELOGE(FAILED, "[Get][SubGraph] Node %s has no subgraph, index is %u.", node->GetName().c_str(), cond_index_new); return ge::FAILED; } auto chosen_graph = GraphUtils::FindRootGraph(node->GetOwnerComputeGraph())->GetSubgraph(chosen_branch_name); @@ -132,7 +132,7 @@ Status CondRemovePass::GetIfChosenBranch(const NodePtr &node, const uint32_t con if (subgraph_names_size == 0) { REPORT_INNER_ERROR("E19999", "subgraph size of op:%s(%s) is 0, check invavlid", node->GetName().c_str(), node->GetType().c_str()); - GELOGE(FAILED, "Node %s has none subgraph.", node->GetName().c_str()); + GELOGE(FAILED, "[Check][Param] Node %s has none subgraph.", node->GetName().c_str()); return ge::FAILED; } // If cond is false, else branch @@ -143,7 +143,7 @@ Status CondRemovePass::GetIfChosenBranch(const NodePtr &node, const uint32_t con if (chosen_branch_name.empty()) { REPORT_INNER_ERROR("E19999", "Get subgraph name from op:%s(%s) by index:%u failed", node->GetName().c_str(), node->GetType().c_str(), cond_index_new); - GELOGE(FAILED, "Node %s has no subgraph, index is %u.", node->GetName().c_str(), cond_index_new); + GELOGE(FAILED, "[Get][SubGraph] Node %s has no subgraph, index is %u.", node->GetName().c_str(), cond_index_new); return ge::FAILED; } auto chosen_graph = GraphUtils::FindRootGraph(node->GetOwnerComputeGraph())->GetSubgraph(chosen_branch_name); @@ -151,7 +151,7 @@ Status CondRemovePass::GetIfChosenBranch(const NodePtr &node, const uint32_t con REPORT_INNER_ERROR("E19999", "Find subgraph by name:%s from node:%s(%s)'s root_graph failed", chosen_branch_name.c_str(), node->GetName().c_str(), node->GetType().c_str()); - GELOGE(FAILED, "Can not find branch %s in node %s's parent graph %s.", chosen_branch_name.c_str(), + GELOGE(FAILED, "[Check][Param] Can not find branch %s in node %s's parent graph %s.", chosen_branch_name.c_str(), node->GetName().c_str(), node->GetOwnerComputeGraph()->GetName().c_str()); return ge::FAILED; } @@ -263,7 +263,7 @@ Status CondRemovePass::ReplaceIfCaseNodeWithPartitioncall(const NodePtr &node, c peerout_anchor->GetOwnerNode()->GetType().c_str(), peerout_anchor->GetIdx(), partitioncall_node->GetName().c_str(), partitioncall_node->GetType().c_str(), input_anchor->GetIdx()); - GELOGE(FAILED, "Add edge failed, from node:%s idx:%d to node:%s idx:%d, input num:%zu, output num:%zu", + GELOGE(FAILED, "[Add][Edge] failed, from node:%s idx:%d to node:%s idx:%d, input num:%zu, output num:%zu", peerout_anchor->GetOwnerNode()->GetName().c_str(), peerout_anchor->GetIdx(), partitioncall_node->GetName().c_str(), input_anchor->GetIdx(), input_desc_size, output_desc_size); @@ -280,7 +280,7 @@ Status CondRemovePass::ReplaceIfCaseNodeWithPartitioncall(const NodePtr &node, c node->GetName().c_str(), node->GetType().c_str(), output_anchor->GetIdx(), peerin_anchor->GetOwnerNode()->GetName().c_str(), peerin_anchor->GetOwnerNode()->GetType().c_str(), peerin_anchor->GetIdx()); - GELOGE(FAILED, "Remove edge failed, from node:%s idx:%d to node:%s idx:%d, input num:%zu, output num:%zu", + GELOGE(FAILED, "[Remove][Edge] failed, from node:%s idx:%d to node:%s idx:%d, input num:%zu, output num:%zu", node->GetName().c_str(), output_anchor->GetIdx(), peerin_anchor->GetOwnerNode()->GetName().c_str(), peerin_anchor->GetIdx(), input_desc_size, output_desc_size); return FAILED; @@ -292,7 +292,7 @@ Status CondRemovePass::ReplaceIfCaseNodeWithPartitioncall(const NodePtr &node, c partitioncall_node->GetType().c_str(), output_anchor->GetIdx(), peerin_anchor->GetOwnerNode()->GetName().c_str(), peerin_anchor->GetOwnerNode()->GetType().c_str(), peerin_anchor->GetIdx()); - GELOGE(FAILED, "Add edge failed, from node:%s idx:%d to node:%s idx:%d, input num:%zu, output num:%zu", + GELOGE(FAILED, "[Add][Edge] failed, from node:%s idx:%d to node:%s idx:%d, input num:%zu, output num:%zu", partitioncall_node->GetName().c_str(), output_anchor->GetIdx(), peerin_anchor->GetOwnerNode()->GetName().c_str(), peerin_anchor->GetIdx(), input_desc_size, output_desc_size); @@ -370,7 +370,7 @@ Status CondRemovePass::GetCondInfo(const NodePtr &node, ComputeGraphPtr &graph, std::string type = node->GetType(); if ((kIfOpTypes.count(type) != 0) || (kCaseOpTypes.count(type) != 0)) { if (GetCondInfoForIfCase(node, graph, cond_out_anchor, cond_in_anchor) != SUCCESS) { - GELOGE(FAILED, "Get cond_info for if/case node failed."); + GELOGE(FAILED, "[Get][CondInfo] for if/case node:%s failed.", node->GetName().c_str()); return FAILED; } } else { diff --git a/ge/graph/passes/constant_folding_pass.cc b/ge/graph/passes/constant_folding_pass.cc index db2ef494..25fe26da 100644 --- a/ge/graph/passes/constant_folding_pass.cc +++ b/ge/graph/passes/constant_folding_pass.cc @@ -33,12 +33,12 @@ Status RunOpKernelWithCheck(NodePtr &node, std::vector &outputs) { std::shared_ptr instance_ptr = ge::GELib::GetInstance(); if ((instance_ptr == nullptr) || (!instance_ptr->InitFlag())) { - GELOGE(GE_CLI_GE_NOT_INITIALIZED, "GE is not initialized or is finalized."); + GELOGE(GE_CLI_GE_NOT_INITIALIZED, "[Check][Param] GE is not initialized or is finalized."); return UNSUPPORTED; } OpsKernelInfoStorePtr kernel_info = instance_ptr->OpsKernelManagerObj().GetOpsKernelInfoStore(kKernelLibName); if (kernel_info == nullptr) { - GELOGE(FAILED, "Get op kernel info store %s failed", kKernelLibName.c_str()); + GELOGE(FAILED, "[Get][OpsKernelInfoStore] %s failed", kKernelLibName.c_str()); return UNSUPPORTED; } @@ -110,7 +110,7 @@ Status ConstantFoldingPass::Run(ge::NodePtr &node) { } REPORT_CALL_ERROR("E19999", "Calculate for node %s(%s) failed", node->GetName().c_str(), node->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Calculate for node %s failed in constant folding", node->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Call][Calculate] for node %s failed in constant folding", node->GetName().c_str()); return ret; } GELOGI("Node %s type %s, constant folding compute success.", node->GetName().c_str(), node->GetType().c_str()); @@ -129,10 +129,8 @@ Status ConstantFoldingPass::Run(ge::NodePtr &node) { if (outputs.empty()) { REPORT_INNER_ERROR("E19999", "After calculate for node %s(%s), output weight is empty, check invalid", node->GetName().c_str(), node->GetType().c_str()); - GELOGE(INTERNAL_ERROR, - "Failed to constant folding on node %s," - " no output weight", - node->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Check][Param] After calculate for node %s(%s), output weight is empty", + node->GetName().c_str(), node->GetType().c_str()); return INTERNAL_ERROR; } diff --git a/ge/graph/passes/constant_fuse_same_pass.cc b/ge/graph/passes/constant_fuse_same_pass.cc index 8cb8c091..08f5ac4a 100644 --- a/ge/graph/passes/constant_fuse_same_pass.cc +++ b/ge/graph/passes/constant_fuse_same_pass.cc @@ -58,7 +58,7 @@ void GetOutDataNodeToIndexMap(NodePtr &node, std::map & Status ConstantFuseSamePass::Run(ge::ComputeGraphPtr graph) { if (graph == nullptr) { REPORT_INNER_ERROR("E19999", "Param graph is nullptr, check invalid"); - GELOGE(GE_GRAPH_PARAM_NULLPTR, "Compute graph is null."); + GELOGE(GE_GRAPH_PARAM_NULLPTR, "[Check][Param] Compute graph is nullptr."); return GE_GRAPH_PARAM_NULLPTR; } GELOGI("ConstantFuseSamePass in."); @@ -160,13 +160,14 @@ Status ConstantFuseSamePass::MoveOutDataEdges(NodePtr &src_node, NodePtr &dst_no } auto ret = dst_out_data_anchor->LinkTo(it->second); if (ret != SUCCESS) { - REPORT_CALL_ERROR("E19999", - "Op:%s(%s) out index:0 link to op:%s(%s) in index:%d failed", + REPORT_CALL_ERROR("E19999", "Op:%s(%s) out index:0 link to op:%s(%s) in index:%d failed", dst_node->GetName().c_str(), dst_node->GetType().c_str(), it->second->GetOwnerNode()->GetName().c_str(), it->second->GetOwnerNode()->GetType().c_str(), it->second->GetIdx()); - GELOGE(FAILED, "Failed to move out data edge from %s to %s", src_node->GetName().c_str(), - dst_node->GetName().c_str()); + GELOGE(FAILED, "[Add][Edge] Op:%s(%s) out index:0 link to op:%s(%s) in index:%d failed", + dst_node->GetName().c_str(), dst_node->GetType().c_str(), + it->second->GetOwnerNode()->GetName().c_str(), it->second->GetOwnerNode()->GetType().c_str(), + it->second->GetIdx()); return FAILED; } } @@ -193,7 +194,8 @@ Status ConstantFuseSamePass::FuseConstNodes(ComputeGraphPtr &graph, if (GraphUtils::RemoveNodeWithoutRelink(graph, node) != SUCCESS) { REPORT_CALL_ERROR("E19999", "Remove node:%s(%s) without relink in graph:%s failed", node->GetName().c_str(), node->GetType().c_str(), graph->GetName().c_str()); - GELOGE(FAILED, "[%s] RemoveNodeWithoutRelink failed.", node->GetName().c_str()); + GELOGE(FAILED, "[Remove][Node] %s(%s) Without Relink in graph:%s failed.", + node->GetName().c_str(), node->GetType().c_str(), graph->GetName().c_str()); return FAILED; } } diff --git a/ge/graph/passes/control_trigger_pass.cc b/ge/graph/passes/control_trigger_pass.cc index 9125a48f..85505dc5 100644 --- a/ge/graph/passes/control_trigger_pass.cc +++ b/ge/graph/passes/control_trigger_pass.cc @@ -30,7 +30,7 @@ Status ControlTriggerPass::Run(ComputeGraphPtr graph) { auto in_ctrl_nodes = node->GetInControlNodes(); for (NodePtr &in_ctrl_node : in_ctrl_nodes) { if (HandleDynamicCtrlEdges(graph, node, in_ctrl_node) != SUCCESS) { - GELOGE(FAILED, "HandleDynamicCtrlEdges for %s->%s fail.", in_ctrl_node->GetName().c_str(), + GELOGE(FAILED, "[Handle][DynamicCtrlEdges] for node:%s->node:%s failed.", in_ctrl_node->GetName().c_str(), node->GetName().c_str()); return FAILED; } @@ -55,7 +55,7 @@ Status ControlTriggerPass::HandleDynamicCtrlEdges(ComputeGraphPtr &graph, NodePt NodePtr switch_node = nullptr; bool branch_flag = false; if (FindSwitchNode(in_ctrl_node, switch_node, branch_flag) != SUCCESS) { - GELOGE(FAILED, "FindSwitchNode fail."); + GELOGE(FAILED, "[Find][SwitchNode] failed, in_ctrl_node:%s.", in_ctrl_node->GetName().c_str()); return FAILED; } @@ -76,8 +76,9 @@ Status ControlTriggerPass::HandleDynamicCtrlEdges(ComputeGraphPtr &graph, NodePt node->GetName().c_str(), node->GetType().c_str(), in_ctrl_node->GetName().c_str(), in_ctrl_node->GetType().c_str(), constant->GetName().c_str(), constant->GetType().c_str()); - GELOGE(FAILED, "Replace ctrl edge fail, %s->%s, %s->%s.", in_ctrl_node->GetName().c_str(), - node->GetName().c_str(), in_ctrl_node->GetName().c_str(), constant->GetName().c_str()); + GELOGE(FAILED, "[Replace][CtrlEdge] failed, remove edge:%s->%s, add edge:%s->%s.", + in_ctrl_node->GetName().c_str(), node->GetName().c_str(), + in_ctrl_node->GetName().c_str(), constant->GetName().c_str()); return FAILED; } @@ -87,7 +88,8 @@ Status ControlTriggerPass::HandleDynamicCtrlEdges(ComputeGraphPtr &graph, NodePt } if (InsertOppositeBranch(graph, node, in_ctrl_node, switch_node, branch_flag) != SUCCESS) { - GELOGE(FAILED, "InsertOppositeBranch fail."); + GELOGE(FAILED, "[Insert][OppositeBranch] failed, node:%s, in_ctrl_node:%s.", + node->GetName().c_str(), in_ctrl_node->GetName().c_str()); return FAILED; } @@ -145,7 +147,7 @@ Status ControlTriggerPass::FindSwitchNode(const NodePtr &node, NodePtr &switch_n case kNotControlOp: break; default: - GELOGE(FAILED, "invalid type"); + GELOGE(FAILED, "[Check][Param] invalid node type"); return FAILED; } @@ -167,6 +169,12 @@ Status ControlTriggerPass::FindSwitchNode(const NodePtr &node, NodePtr &switch_n /// @return ControlNodeType /// ControlNodeType ControlTriggerPass::TransferNodeType(const NodePtr &node, uint32_t index) { + OpDescPtr merge_desc = node->GetOpDesc(); + if (merge_desc == nullptr) { + REPORT_INNER_ERROR("E19999", "op_desc in merge node is nullptr, check invalid"); + GELOGE(INTERNAL_ERROR, "[Get][OpDesc] failed, merge_desc is nullptr."); + return kInvalidType; + } const std::string type = node->GetType(); if ((type == SWITCH) || (type == REFSWITCH)) { if ((index != SWITCH_TRUE_OUTPUT) && (index != SWITCH_FALSE_OUTPUT)) { @@ -175,7 +183,7 @@ ControlNodeType ControlTriggerPass::TransferNodeType(const NodePtr &node, uint32 } if (FindPredInput(node) != SUCCESS) { - GELOGE(INTERNAL_ERROR, "FindPredInput fail, switch_node: %s.", node->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Find][PredInput] failed, switch_node:%s.", node->GetName().c_str()); return kInvalidType; } @@ -189,12 +197,6 @@ ControlNodeType ControlTriggerPass::TransferNodeType(const NodePtr &node, uint32 return branch_flag ? kLoopSwitchT : kLoopSwitchF; } } else if ((type == MERGE) || (type == REFMERGE)) { - OpDescPtr merge_desc = node->GetOpDesc(); - if (merge_desc == nullptr) { - REPORT_INNER_ERROR("E19999", "op_desc in merge node is nullptr, check invalid"); - GELOGE(INTERNAL_ERROR, "FindPredInput fail, merge_desc is null, merge_node: %s.", node->GetName().c_str()); - return kInvalidType; - } if (!merge_desc->HasAttr(ATTR_NAME_NEXT_ITERATION)) { return kCondMerge; } @@ -248,14 +250,16 @@ Status ControlTriggerPass::InsertOppositeBranch(ComputeGraphPtr &graph, NodePtr NodePtr merge_node = InsertMergeNode(graph, node, in_ctrl_node, data_desc); if (merge_node == nullptr) { - GELOGE(FAILED, "InsertMergeNode fail."); + GELOGE(FAILED, "[Insert][MergeNode] failed, node:%s, in_ctrl_node:%s.", + node->GetName().c_str(), in_ctrl_node->GetName().c_str()); return FAILED; } NodePtr const_f = InsertConstNode(graph, merge_node, data_desc, false); NodePtr const_t = InsertConstNode(graph, merge_node, data_desc, true); if ((const_f == nullptr) || (const_t == nullptr)) { - GELOGE(FAILED, "InsertConstNode fail."); + GELOGE(FAILED, "[Insert][ConstNode] failed, graph:%s, merge_node:%s.", + graph->GetName().c_str(), merge_node->GetName().c_str()); return FAILED; } @@ -266,7 +270,8 @@ Status ControlTriggerPass::InsertOppositeBranch(ComputeGraphPtr &graph, NodePtr const std::string identity_name = switch_desc->GetName() + "_" + IDENTITY; NodePtr identity_node = InsertIdentityNode(graph, identity_name, switch_desc->GetOutputDesc(new_idx)); if (identity_node == nullptr) { - GELOGE(FAILED, "InsertIdentityNode fail."); + GELOGE(FAILED, "[Insert][IdentityNode] name:%s failed, graph:%s.", + identity_name.c_str(), graph->GetName().c_str()); return FAILED; } @@ -274,21 +279,25 @@ Status ControlTriggerPass::InsertOppositeBranch(ComputeGraphPtr &graph, NodePtr REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed", in_ctrl_node->GetName().c_str(), in_ctrl_node->GetType().c_str(), orig_const->GetName().c_str(), orig_const->GetType().c_str()); - GELOGE(FAILED, "Add in ctrl edge fail, %s->%s.", in_ctrl_node->GetName().c_str(), orig_const->GetName().c_str()); + GELOGE(FAILED, "[Add][CtrlEdge] failed, %s->%s.", in_ctrl_node->GetName().c_str(), orig_const->GetName().c_str()); return FAILED; } if (GraphUtils::AddEdge(switch_node->GetOutDataAnchor(new_idx), identity_node->GetInDataAnchor(0)) != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(index:%u) and op:%s(%s)(index:0) failed", switch_node->GetName().c_str(), switch_node->GetType().c_str(), new_idx, identity_node->GetName().c_str(), identity_node->GetType().c_str()); - GELOGE(FAILED, "Add in data edge fail, %s->%s.", switch_desc->GetName().c_str(), identity_node->GetName().c_str()); + GELOGE(FAILED, "[Add][Edge] between op:%s(%s)(index:%u) and op:%s(%s)(index:0) failed", + switch_node->GetName().c_str(), switch_node->GetType().c_str(), new_idx, + identity_node->GetName().c_str(), identity_node->GetType().c_str()); return FAILED; } if (GraphUtils::AddEdge(identity_node->GetOutControlAnchor(), new_const->GetInControlAnchor()) != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed", identity_node->GetName().c_str(), identity_node->GetType().c_str(), new_const->GetName().c_str(), new_const->GetType().c_str()); - GELOGE(FAILED, "Add in ctrl edge fail, %s->%s.", identity_node->GetName().c_str(), new_const->GetName().c_str()); + GELOGE(FAILED, "[Add][ControlEdge] between op:%s(%s) and op:%s(%s) failed", + identity_node->GetName().c_str(), identity_node->GetType().c_str(), + new_const->GetName().c_str(), new_const->GetType().c_str()); return FAILED; } @@ -299,7 +308,7 @@ Status ControlTriggerPass::InsertOppositeBranch(ComputeGraphPtr &graph, NodePtr } else { if (!iter->second.insert(pred_const).second) { REPORT_INNER_ERROR("E19999", "Insert to control_trigger_map_ failed"); - GELOGE(FAILED, "control_trigger_map_ insert failed."); + GELOGE(FAILED, "[Check][Param] control_trigger_map_ insert failed."); return FAILED; } } @@ -321,7 +330,7 @@ NodePtr ControlTriggerPass::InsertMergeNode(ComputeGraphPtr &graph, NodePtr &nod OpDescPtr op_desc = MakeShared(name, MERGE); if (op_desc == nullptr) { REPORT_CALL_ERROR("E19999", "New OpDesc failed"); - GELOGE(FAILED, "Create Merge op %s: create op_desc fail.", name.c_str()); + GELOGE(FAILED, "[New][OpDesc] failed"); return nullptr; } @@ -329,7 +338,8 @@ NodePtr ControlTriggerPass::InsertMergeNode(ComputeGraphPtr &graph, NodePtr &nod (op_desc->AddOutputDesc(data_desc) != GRAPH_SUCCESS) || (op_desc->AddOutputDesc(data_desc) != GRAPH_SUCCESS)) { REPORT_CALL_ERROR("E19999", "Add input or ouput desc to op:%s(%s) failed", op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Create Merge op %s: add input/output desc fail.", name.c_str()); + GELOGE(INTERNAL_ERROR, "[Add][GeTensorDesc] to op:%s(%s) failed", + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return nullptr; } @@ -338,7 +348,8 @@ NodePtr ControlTriggerPass::InsertMergeNode(ComputeGraphPtr &graph, NodePtr &nod if (merge_node == nullptr) { REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s failed", op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "Create Merge op %s fail.", name.c_str()); + GELOGE(INTERNAL_ERROR, "[Add][Node] %s(%s) to graph:%s failed", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str()); return nullptr; } @@ -350,7 +361,8 @@ NodePtr ControlTriggerPass::InsertMergeNode(ComputeGraphPtr &graph, NodePtr &nod node->GetName().c_str(), node->GetType().c_str(), merge_node->GetName().c_str(), merge_node->GetType().c_str(), node->GetName().c_str(), node->GetType().c_str()); - GELOGE(FAILED, "Replace ctrl edge fail, %s->%s, %s->%s", in_ctrl_node->GetName().c_str(), node->GetName().c_str(), + GELOGE(FAILED, "[Replace][CtrlEdge] failed, remove edge:%s->%s, add edge:%s->%s", + in_ctrl_node->GetName().c_str(), node->GetName().c_str(), merge_node->GetName().c_str(), node->GetName().c_str()); return nullptr; } @@ -372,7 +384,7 @@ NodePtr ControlTriggerPass::InsertConstNode(ComputeGraphPtr &graph, NodePtr &mer OpDescPtr op_desc = MakeShared(name, CONSTANT); if (op_desc == nullptr) { REPORT_CALL_ERROR("E19999", "New OpDesc failed"); - GELOGE(FAILED, "Create Const op %s: create op_desc fail.", name.c_str()); + GELOGE(FAILED, "[New][OpDesc] failed."); return nullptr; } @@ -380,20 +392,22 @@ NodePtr ControlTriggerPass::InsertConstNode(ComputeGraphPtr &graph, NodePtr &mer GeTensorPtr const_value = MakeShared(data_desc, reinterpret_cast(&value), sizeof(int32_t)); if (const_value == nullptr) { REPORT_CALL_ERROR("E19999", "New GeTensor failed"); - GELOGE(FAILED, "Create tensor fail."); + GELOGE(FAILED, "[New][GeTensor] failed."); return nullptr; } if (!AttrUtils::SetTensor(op_desc, ATTR_NAME_WEIGHTS, const_value)) { REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed", ATTR_NAME_WEIGHTS.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Create Const op %s: set attr ATTR_NAME_WEIGHTS fail.", name.c_str()); + GELOGE(INTERNAL_ERROR, "[Set][Attr] %s to op:%s(%s) failed", ATTR_NAME_WEIGHTS.c_str(), + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return nullptr; } if (op_desc->AddOutputDesc(data_desc) != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Add ouput desc to op:%s(%s) failed", op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Create Const op %s: add output desc fail.", name.c_str()); + GELOGE(INTERNAL_ERROR, "[Add][OutputDesc] to op:%s(%s) failed", + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return nullptr; } @@ -402,7 +416,8 @@ NodePtr ControlTriggerPass::InsertConstNode(ComputeGraphPtr &graph, NodePtr &mer if (const_node == nullptr) { REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s failed", op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "Create Const op %s fail.", name.c_str()); + GELOGE(INTERNAL_ERROR, "[Add][Node] %s(%s) to graph:%s failed", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str()); return nullptr; } @@ -411,7 +426,9 @@ NodePtr ControlTriggerPass::InsertConstNode(ComputeGraphPtr &graph, NodePtr &mer REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(index:0) and op:%s(%s)(index:%u) failed", const_node->GetName().c_str(), const_node->GetType().c_str(), merge_node->GetName().c_str(), merge_node->GetType().c_str(), out_idx); - GELOGE(FAILED, "Add in data edge fail, %s->%s", const_node->GetName().c_str(), merge_node->GetName().c_str()); + GELOGE(FAILED, "[Add][Edge] between op:%s(%s)(index:0) and op:%s(%s)(index:%u) failed", + const_node->GetName().c_str(), const_node->GetType().c_str(), + merge_node->GetName().c_str(), merge_node->GetType().c_str(), out_idx); return nullptr; } @@ -430,14 +447,15 @@ NodePtr ControlTriggerPass::InsertIdentityNode(ComputeGraphPtr &graph, const std OpDescPtr op_desc = MakeShared(name, IDENTITY); if (op_desc == nullptr) { REPORT_CALL_ERROR("E19999", "New OpDesc failed"); - GELOGE(FAILED, "Create Identity op %s: create op_desc fail.", name.c_str()); + GELOGE(FAILED, "[New][OpDesc] failed"); return nullptr; } if ((op_desc->AddInputDesc(data_desc) != GRAPH_SUCCESS) || (op_desc->AddOutputDesc(data_desc) != GRAPH_SUCCESS)) { REPORT_CALL_ERROR("E19999", "Add input or output desc to op:%s(%s) failed", op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Create Identity op %s: add input/output desc fail.", name.c_str()); + GELOGE(INTERNAL_ERROR, "[Add][GeTensorDesc] to op:%s(%s) failed", + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return nullptr; } @@ -446,7 +464,8 @@ NodePtr ControlTriggerPass::InsertIdentityNode(ComputeGraphPtr &graph, const std if (identity_node == nullptr) { REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s failed", op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "Create Identity op %s fail.", name.c_str()); + GELOGE(INTERNAL_ERROR, "[Add][Node] %s(%s) to graph:%s failed", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str()); return nullptr; } @@ -463,7 +482,7 @@ NodePtr ControlTriggerPass::InsertIdentityNode(ComputeGraphPtr &graph, const std Status ControlTriggerPass::FindPredInput(const NodePtr &switch_node) { if (switch_node == nullptr) { REPORT_INNER_ERROR("E19999", "Param switch_node is nullptr, check invalid"); - GELOGE(INTERNAL_ERROR, "switch_node is null"); + GELOGE(INTERNAL_ERROR, "[Check][Param] switch_node is nullptr"); return INTERNAL_ERROR; } @@ -472,7 +491,8 @@ Status ControlTriggerPass::FindPredInput(const NodePtr &switch_node) { REPORT_INNER_ERROR("E19999", "Index:%d in anchor of switch_node:%s(%s) is nullptr, check invalid", SWITCH_PRED_INPUT, switch_node->GetName().c_str(), switch_node->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "in_cond_anchor is nullptr, node: %s.", switch_node->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Get][InDataAnchor] Index:%d in anchor of switch_node:%s(%s) is nullptr", + SWITCH_PRED_INPUT, switch_node->GetName().c_str(), switch_node->GetType().c_str()); return INTERNAL_ERROR; } OutDataAnchorPtr pred_cond_anchor = in_cond_anchor->GetPeerOutAnchor(); @@ -480,7 +500,8 @@ Status ControlTriggerPass::FindPredInput(const NodePtr &switch_node) { REPORT_INNER_ERROR("E19999", "Index:%d in anchor of switch_node:%s(%s), it's peer anchor is nullptr, " "check invalid", SWITCH_PRED_INPUT, switch_node->GetName().c_str(), switch_node->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "pred_cond_anchor is nullptr, node: %s.", switch_node->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "Index:%d in anchor of switch_node:%s(%s), it's peer anchor is nullptr", + SWITCH_PRED_INPUT, switch_node->GetName().c_str(), switch_node->GetType().c_str()); return INTERNAL_ERROR; } diff --git a/ge/graph/passes/ctrl_edge_transfer_pass.cc b/ge/graph/passes/ctrl_edge_transfer_pass.cc index 598d2e14..f1796b2f 100755 --- a/ge/graph/passes/ctrl_edge_transfer_pass.cc +++ b/ge/graph/passes/ctrl_edge_transfer_pass.cc @@ -65,13 +65,17 @@ Status CtrlEdgeTransferPass::Run(ge::ComputeGraphPtr graph) { for (auto &in_control_node : n->GetInControlNodes()) { GE_CHECK_NOTNULL(in_control_node); GE_CHK_GRAPH_STATUS_RET(ge::GraphUtils::RemoveEdge(in_control_node->GetOutControlAnchor(), - n->GetInControlAnchor()), "remove edge failed"); + n->GetInControlAnchor()), + "[Remove][ControlEdge] between %s and %s failed", + in_control_node->GetName().c_str(), n->GetName().c_str()); for (auto &out_node : n->GetOutNodes()) { if (out_node == nullptr) { continue; } GE_CHK_GRAPH_STATUS_RET(ge::GraphUtils::AddEdge(in_control_node->GetOutControlAnchor(), - out_node->GetInControlAnchor()), "add edge failed."); + out_node->GetInControlAnchor()), + "[Add][ControlEdge] between %s and %s failed.", + in_control_node->GetName().c_str(), out_node->GetName().c_str()); } } } diff --git a/ge/graph/passes/data_pass.cc b/ge/graph/passes/data_pass.cc index cb94b161..0974baf2 100644 --- a/ge/graph/passes/data_pass.cc +++ b/ge/graph/passes/data_pass.cc @@ -32,7 +32,8 @@ Status MappingSubgraphInput(const ComputeGraphPtr &graph, const std::functionGetOpDesc(), "index", index)) { REPORT_CALL_ERROR("E19999", "Get Attr:%s from op:%s(%s) failed", "index", node->GetName().c_str(), node->GetType().c_str()); - GELOGE(FAILED, "Failed to get index from data[%s]", node->GetName().c_str()); + GELOGE(FAILED, "[Get][Attr] index from op:%s(%s) failed", + node->GetName().c_str(), node->GetType().c_str()); return FAILED; } @@ -42,7 +43,8 @@ Status MappingSubgraphInput(const ComputeGraphPtr &graph, const std::functionGetOpDesc(), ATTR_NAME_PARENT_NODE_INDEX, parent_index)) { REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed", ATTR_NAME_PARENT_NODE_INDEX.c_str(), node->GetName().c_str(), node->GetType().c_str()); - GELOGE(FAILED, "Failed to set parent index for node %s", node->GetName().c_str()); + GELOGE(FAILED, "[Set][Attr] %s to op:%s(%s) failed", ATTR_NAME_PARENT_NODE_INDEX.c_str(), + node->GetName().c_str(), node->GetType().c_str()); return FAILED; } } @@ -72,7 +74,8 @@ Status MappingSubgraphOutput(const ComputeGraphPtr &graph, const std::functionGetName().c_str(), op_desc->GetType().c_str(), index); - GELOGE(FAILED, "Failed to set parent index for graph %s", graph->GetName().c_str()); + GELOGE(FAILED, "[Set][Attr] %s to tensor of op:%s(%s) input:%zu failed", + ATTR_NAME_PARENT_NODE_INDEX.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str(), index); return FAILED; } } @@ -87,12 +90,12 @@ Status MappingSubgraphIndex(const ComputeGraphPtr &graph, GE_CHECK_NOTNULL(input); GE_CHECK_NOTNULL(output); if (MappingSubgraphInput(graph, input) != SUCCESS) { - GELOGE(FAILED, "Failed to mapping subgraph input for graph: %s", graph->GetName().c_str()); + GELOGE(FAILED, "[Call][MappingSubgraphInput] for graph:%s failed", graph->GetName().c_str()); return FAILED; } if (MappingSubgraphOutput(graph, output) != SUCCESS) { - GELOGE(FAILED, "Failed to mapping subgraph output for graph: %s", graph->GetName().c_str()); + GELOGE(FAILED, "[Call][MappingSubgraphOutput] for graph:%s failed", graph->GetName().c_str()); return FAILED; } @@ -149,7 +152,7 @@ Status DataPass::PostParseSubgraph(const ComputeGraphPtr &graph, const string &i if (post_func_it == subgraph_handle.end()) { REPORT_INNER_ERROR("E19999", "The subgraph post func for node %s type %s is null, check invalid", parent_node->GetName().c_str(), parent_node->GetType().c_str()); - GELOGE(FAILED, "The subgraph post func for node %s type %s is null.", + GELOGE(FAILED, "[Check][Param] The subgraph post func for node %s type %s is null.", parent_node->GetName().c_str(), parent_node->GetType().c_str()); return FAILED; } @@ -157,7 +160,7 @@ Status DataPass::PostParseSubgraph(const ComputeGraphPtr &graph, const string &i if (post_func_it->second(ir_name, graph) != SUCCESS) { REPORT_INNER_ERROR("E19999", "Post process subgraph %s on node %s type %s failed", graph->GetName().c_str(), parent_node->GetName().c_str(), parent_node->GetType().c_str()); - GELOGE(FAILED, "Failed to post process subgraph %s on node %s type %s", + GELOGE(FAILED, "[Call][PostFunc] Failed to post process subgraph %s on node %s type %s", graph->GetName().c_str(), parent_node->GetName().c_str(), parent_node->GetType().c_str()); return FAILED; } @@ -188,7 +191,7 @@ Status DataPass::Run(ComputeGraphPtr compute_graph) { GE_CHECK_NOTNULL(parent_node->GetOpDesc()); auto func_desc = parent_node->GetOpDesc(); GE_CHK_STATUS_RET(func_desc->GetSubgraphNameByInstanceName(compute_graph->GetName(), subgraph_name), - "Subgraph: %s get subgraph name failed.", compute_graph->GetName().c_str()); + "[Get][SubGraphName] for Graph:%s failed.", compute_graph->GetName().c_str()); GELOGI("Post process for subgraph %s, Subgraph name: %s, Parent name: %s, Parent type: %s.", compute_graph->GetName().c_str(), subgraph_name.c_str(), parent_node->GetName().c_str(), diff --git a/ge/graph/passes/dimension_adjust_pass.cc b/ge/graph/passes/dimension_adjust_pass.cc index dbea8dc9..03cf3053 100755 --- a/ge/graph/passes/dimension_adjust_pass.cc +++ b/ge/graph/passes/dimension_adjust_pass.cc @@ -30,14 +30,14 @@ const int kRemoveInputIndex = 1; Status DimensionAdjustPass::Run(ge::NodePtr &node) { if (node == nullptr) { REPORT_INNER_ERROR("E19999", "Param node is nullptr, check invalid"); - GELOGE(PARAM_INVALID, "node is nullptr."); + GELOGE(PARAM_INVALID, "[Check][Param] node is nullptr."); return PARAM_INVALID; } OpDescPtr op_desc_ptr = node->GetOpDesc(); if (op_desc_ptr == nullptr) { REPORT_INNER_ERROR("E19999", "Param op_desc of node is nullptr, check invalid"); - GELOGE(PARAM_INVALID, "GetOpDesc return nullptr."); + GELOGE(PARAM_INVALID, "[Get][OpDesc] return nullptr."); return PARAM_INVALID; } @@ -46,7 +46,7 @@ Status DimensionAdjustPass::Run(ge::NodePtr &node) { if (ret != SUCCESS) { REPORT_CALL_ERROR("E19999", "Get OriginalType of op:%s(%s) failed", node->GetName().c_str(), node->GetType().c_str()); - GELOGE(ret, "DimensionAdjustPass get originnal type fail."); + GELOGE(ret, "[Get][OriginalType] of op:%s(%s) failed", node->GetName().c_str(), node->GetType().c_str()); return ret; } @@ -75,7 +75,7 @@ Status DimensionAdjustPass::Run(ge::NodePtr &node) { } REPORT_CALL_ERROR("E19999", "kernel compute for op:%s(%s) failed", node->GetName().c_str(), node->GetType().c_str()); - GELOGE(ret, "DimensionAdjustPass compute failed"); + GELOGE(ret, "[Call][Compute] for op:%s(%s) failed", node->GetName().c_str(), node->GetType().c_str()); return ret; } // Need to handle axis_input of node like ExpandDims @@ -88,21 +88,22 @@ Status DimensionAdjustPass::Run(ge::NodePtr &node) { if (ret != SUCCESS) { REPORT_CALL_ERROR("E19999", "Unlink op:%s(%s) data input:%u with control edge copy failed", node->GetName().c_str(), node->GetType().c_str(), kRemoveInputIndex); - GELOGE(ret, "DimensionAdjustPass unlink node with control copy fail."); + GELOGE(ret, "[Unlink][Op] %s(%s) data input:%u with control edge copy failed", + node->GetName().c_str(), node->GetType().c_str(), kRemoveInputIndex); return ret; } // 2.Remove const axis node without any output if ((axis_node->GetType() == CONSTANT || axis_node->GetType() == CONSTANTOP) && axis_node->GetOutDataNodesSize() == 0) { ret = IsolateAndDeleteNode(axis_node, {}); - GE_CHK_GRAPH_STATUS_RET(ret, "Fail to remove node %s.", axis_node->GetName().c_str()); + GE_CHK_GRAPH_STATUS_RET(ret, "[Remove][Node] %s failed.", axis_node->GetName().c_str()); GELOGI("Remove useless axis input const %s", axis_node->GetName().c_str()); } } ret = DealWithInNodes(node); if (ret != SUCCESS) { - GELOGE(ret, "DealWithInNodes of %s failed.", node->GetName().c_str()); + GELOGE(ret, "[DealWith][InNodes] of node:%s failed.", node->GetName().c_str()); return ret; } @@ -147,14 +148,14 @@ NodePtr DimensionAdjustPass::AddIdentityNodeToGraph(const string &name, const Ge ComputeGraphPtr &graph) { if (graph == nullptr) { REPORT_INNER_ERROR("E19999", "Param graph is nullptr, check invalid"); - GELOGE(INTERNAL_ERROR, "Comput graph ptr is null in creating identity node."); + GELOGE(INTERNAL_ERROR, "[Check][Param] Comput graph ptr is nullptr in creating identity node."); return nullptr; } OpDescPtr desc = MakeShared("", ""); if (desc == nullptr) { REPORT_CALL_ERROR("E19999", "New OpDesc failed"); - GELOGE(MEMALLOC_FAILED, "Failed to create op desc."); + GELOGE(MEMALLOC_FAILED, "[New][OpDesc] failed."); return nullptr; } @@ -165,7 +166,8 @@ NodePtr DimensionAdjustPass::AddIdentityNodeToGraph(const string &name, const Ge if ((ret != GRAPH_SUCCESS) || (ret2 != GRAPH_SUCCESS)) { REPORT_CALL_ERROR("E19999", "Add input or ouput desc to op:%s(%s) failed", desc->GetName().c_str(), desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to add input/output desc in creating identity."); + GELOGE(INTERNAL_ERROR, "[Add][GeTensorDesc] to op:%s(%s) failed", + desc->GetName().c_str(), desc->GetType().c_str()); return nullptr; } diff --git a/ge/graph/passes/dimension_compute_pass.cc b/ge/graph/passes/dimension_compute_pass.cc index cfd978b6..350faf71 100755 --- a/ge/graph/passes/dimension_compute_pass.cc +++ b/ge/graph/passes/dimension_compute_pass.cc @@ -41,7 +41,7 @@ Status DimensionComputePass::Run(ge::NodePtr &node) { } else { REPORT_CALL_ERROR("E19999", "kernel compute for op:%s(%s) failed", node->GetName().c_str(), node->GetType().c_str()); - GELOGE(ret, "DimensionComputePass Compute failed"); + GELOGE(ret, "[Call][Compute] for op:%s(%s) failed", node->GetName().c_str(), node->GetType().c_str()); return ret; } } @@ -49,10 +49,8 @@ Status DimensionComputePass::Run(ge::NodePtr &node) { if (outputs.empty()) { REPORT_INNER_ERROR("E19999", "After compute for node %s(%s), output weight is empty, check invalid", node->GetName().c_str(), node->GetType().c_str()); - GELOGE(INTERNAL_ERROR, - "Failed to compute dims for node %s," - " no output weight", - node->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Check][Param] After compute for node %s(%s), output weight is empty", + node->GetName().c_str(), node->GetType().c_str()); return INTERNAL_ERROR; } diff --git a/ge/graph/passes/dropout_pass.cc b/ge/graph/passes/dropout_pass.cc index 11be74f0..03b35efd 100644 --- a/ge/graph/passes/dropout_pass.cc +++ b/ge/graph/passes/dropout_pass.cc @@ -32,12 +32,12 @@ Status DropOutPass::Run(NodePtr &node) { GELOGD("DropOutPass running"); if (node == nullptr) { REPORT_INNER_ERROR("E19999", "Param node is nullptr, check invalid"); - GELOGE(FAILED, "parameter is null."); + GELOGE(FAILED, "[Check][Param] parameter node is nullptr."); return FAILED; } if (node->GetOpDesc() == nullptr) { REPORT_INNER_ERROR("E19999", "Param op_desc of node is nullptr, check invalid"); - GELOGE(PARAM_INVALID, "param [opDesc] must not be null."); + GELOGE(PARAM_INVALID, "[Get][OpDesc] failed, param [opDesc] must not be null."); return PARAM_INVALID; } std::string op_type = node->GetOpDesc()->GetType(); diff --git a/ge/graph/passes/end_of_sequence_add_control_pass.cc b/ge/graph/passes/end_of_sequence_add_control_pass.cc index 361d4a46..0aee7b03 100755 --- a/ge/graph/passes/end_of_sequence_add_control_pass.cc +++ b/ge/graph/passes/end_of_sequence_add_control_pass.cc @@ -27,7 +27,7 @@ namespace ge { Status EndOfSequenceAddControlPass::Run(ComputeGraphPtr graph) { if (graph == nullptr) { REPORT_INNER_ERROR("E19999", "Param graph is nullptr, check invalid"); - GELOGE(PARAM_INVALID, "param [graph] must not be null."); + GELOGE(PARAM_INVALID, "[Check][Param] param [graph] must not be null."); return PARAM_INVALID; } if (graph->GetParentGraph() != nullptr) { @@ -67,7 +67,8 @@ Status EndOfSequenceAddControlPass::Run(ComputeGraphPtr graph) { // Insert control edge Status status = AddControlEdge(end_of_sequence, target_nodes); if (status != SUCCESS) { - GELOGE(FAILED, "Graph add EndOfSequence op out ctrl edge fail."); + GELOGE(FAILED, "[Add][ControlEdge] Graph add EndOfSequence op:%s out ctrl edge failed.", + end_of_sequence->GetName().c_str()); return FAILED; } GELOGI("EndOfSequenceAddControlPass end."); @@ -83,11 +84,12 @@ Status EndOfSequenceAddControlPass::AddControlEdge(NodePtr &end_of_sequence, std } Status status = GraphUtils::AddEdge(out_ctrl_anchor, in_ctrl_anchor); if (status != GRAPH_SUCCESS) { - REPORT_CALL_ERROR("E19999", - "Add control edge between op:%s(%s) and op:%s(%s) failed", + REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed", end_of_sequence->GetName().c_str(), end_of_sequence->GetType().c_str(), node->GetName().c_str(), node->GetType().c_str()); - GELOGE(FAILED, "Graph add EndOfSequence op out ctrl edge fail, dst node: %s.", node->GetName().c_str()); + GELOGE(FAILED, "[Add][ControlEdge] between op:%s(%s) and op:%s(%s) failed", + end_of_sequence->GetName().c_str(), end_of_sequence->GetType().c_str(), + node->GetName().c_str(), node->GetType().c_str()); return FAILED; } GELOGI("Graph add EndOfSequence op out ctrl edge, dst node: %s.", node->GetName().c_str()); diff --git a/ge/graph/passes/enter_pass.cc b/ge/graph/passes/enter_pass.cc index cde3d6d9..81e44e1f 100644 --- a/ge/graph/passes/enter_pass.cc +++ b/ge/graph/passes/enter_pass.cc @@ -38,7 +38,7 @@ Status EnterPass::Run(NodePtr &node) { // enter node has only one input if (node->GetInDataNodes().empty()) { REPORT_INNER_ERROR("E19999", "Param node in data nodes is empty, check invalid"); - GELOGE(PARAM_INVALID, "enter_node %s has no input", node->GetName().c_str()); + GELOGE(PARAM_INVALID, "[Check][Param] enter_node %s has no input", node->GetName().c_str()); return PARAM_INVALID; } NodePtr in_node = node->GetInDataNodes().at(0); @@ -62,18 +62,19 @@ Status EnterPass::Run(NodePtr &node) { REPORT_CALL_ERROR("E19999", "Remove control edge between op:%s(%s) and op:%s(%s) failed", node->GetName().c_str(), node->GetType().c_str(), out_ctrl_node->GetName().c_str(), out_ctrl_node->GetType().c_str()); - GELOGE(FAILED, "Remove Enter ctrl output fail, %s->%s", node->GetName().c_str(), - out_ctrl_node->GetName().c_str()); + GELOGE(FAILED, "[Remove][ControlEdge] between op:%s(%s) and op:%s(%s) failed", + node->GetName().c_str(), node->GetType().c_str(), + out_ctrl_node->GetName().c_str(), out_ctrl_node->GetType().c_str()); return FAILED; } } } else { if (OptimizeEnterWithOnlyDataOut(node, in_node) != SUCCESS) { - GELOGE(FAILED, "Optimize enter node[%s] with only out data node failed.", node->GetName().c_str()); + GELOGE(FAILED, "[Optimize][EnterNode] [%s] with only out data node failed.", node->GetName().c_str()); return FAILED; } if (UnlinkCtrlEdgeBeforeConst(node) != SUCCESS) { - GELOGE(FAILED, "Unlink control edge before const of node[%s]'s out nodes failed.", node->GetName().c_str()); + GELOGE(FAILED, "[Unlink][ControlEdge] before const of node[%s]'s out nodes failed.", node->GetName().c_str()); return FAILED; } } diff --git a/ge/graph/passes/flow_ctrl_pass.cc b/ge/graph/passes/flow_ctrl_pass.cc index 0072224b..fe0cb9ea 100755 --- a/ge/graph/passes/flow_ctrl_pass.cc +++ b/ge/graph/passes/flow_ctrl_pass.cc @@ -52,7 +52,8 @@ Status FlowCtrlPass::Run(ComputeGraphPtr compute_graph) { // Add big cycle Status ret = AddFpBpIteratorCtrl(compute_graph, node); if (ret != SUCCESS) { - GELOGE(ret, "AddFpBpIteratorCtrl fail, node: %s.", node->GetName().c_str()); + GELOGE(ret, "[Add][FpBpIteratorCtrl] failed, node:%s, graph:%s.", + node->GetName().c_str(), compute_graph->GetName().c_str()); return ret; } graph_change = true; @@ -74,7 +75,8 @@ Status FlowCtrlPass::Run(ComputeGraphPtr compute_graph) { if (is_found && need_cycle_flag) { Status ret = AddSpecialNodeIteratorCtrl(compute_graph, node); if (ret != SUCCESS) { - GELOGE(ret, "AddSpecialNodeIteratorCtrl fail, node: %s.", node->GetName().c_str()); + GELOGE(ret, "[Add][SpecialNodeIteratorCtrl] failed, node:%s, graph:%s.", + node->GetName().c_str(), compute_graph->GetName().c_str()); return ret; } graph_change = true; @@ -116,7 +118,7 @@ NodePtr FlowCtrlPass::InsertOp(ComputeGraphPtr &compute_graph, const string &nod OpDescPtr op_desc = MakeShared(node_name, node_type); if (op_desc == nullptr) { REPORT_CALL_ERROR("E19999", "New OpDesc failed"); - GELOGE(FAILED, "Make OpDesc failed, name:%s, type:%s.", node_name.c_str(), node_type.c_str()); + GELOGE(FAILED, "[New][OpDesc] failed, name:%s, type:%s.", node_name.c_str(), node_type.c_str()); return nullptr; } @@ -125,7 +127,7 @@ NodePtr FlowCtrlPass::InsertOp(ComputeGraphPtr &compute_graph, const string &nod if (graph_status != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Add input desc to op:%s(%s) failed", op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(FAILED, "Add node:%s intput desc failed, error=%u.", node_name.c_str(), graph_status); + GELOGE(FAILED, "[Add][InputDesc] to op:%s(%s) failed", op_desc->GetName().c_str(), op_desc->GetType().c_str()); return nullptr; } } @@ -135,20 +137,22 @@ NodePtr FlowCtrlPass::InsertOp(ComputeGraphPtr &compute_graph, const string &nod if (graph_status != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Add output desc to op:%s(%s) failed", op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(FAILED, "Add node:%s output desc failed, error=%u.", node_name.c_str(), graph_status); + GELOGE(FAILED, "[Add][OutputDesc] to op:%s(%s) failed", + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return nullptr; } } GE_IF_BOOL_EXEC(compute_graph == nullptr, REPORT_INNER_ERROR("E19999", "Param compute_graph is nullptr, check invalid"); - DOMI_LOGE("compute_graph is nullptr"); + DOMI_LOGE("[Check][Param] compute_graph is nullptr"); return nullptr); NodePtr node = compute_graph->AddNode(op_desc); if (node == nullptr) { REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s failed", op_desc->GetName().c_str(), op_desc->GetType().c_str(), compute_graph->GetName().c_str()); - GELOGE(FAILED, "add node failed, name:%s, type:%s.", node_name.c_str(), node_type.c_str()); + GELOGE(FAILED, "[Add][Node] %s(%s) to graph:%s failed", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), compute_graph->GetName().c_str()); return nullptr; } @@ -159,21 +163,19 @@ NodePtr FlowCtrlPass::InsertOp(ComputeGraphPtr &compute_graph, const string &nod NodePtr FlowCtrlPass::InsertStreamSwitchOp(ComputeGraphPtr &compute_graph, const string &switch_name, const NodePtr &loop_cond, const NodePtr &iter_per_loop) { GE_IF_BOOL_EXEC(loop_cond == nullptr || loop_cond->GetOpDesc() == nullptr, - REPORT_INNER_ERROR("E19999", "Param loop_cond or its op_desc is nullptr, " - "check invalid"); - GELOGE(FAILED, "loop_cond is null"); + REPORT_INNER_ERROR("E19999", "Param loop_cond or its op_desc is nullptr, check invalid"); + GELOGE(FAILED, "[Check][Param] Param loop_cond or its op_desc is nullptr"); return nullptr); GE_IF_BOOL_EXEC(iter_per_loop == nullptr || iter_per_loop->GetOpDesc() == nullptr, - REPORT_INNER_ERROR("E19999", "Param iter_per_loop or its op_desc is nullptr, " - "check invalid"); - GELOGE(FAILED, "iter_per_loop is nullptr"); + REPORT_INNER_ERROR("E19999", "Param iter_per_loop or its op_desc is nullptr, check invalid"); + GELOGE(FAILED, "[Check][Param] Param iter_per_loop or its op_desc is nullptr"); return nullptr); std::vector input_desc_list = {loop_cond->GetOpDesc()->GetOutputDesc(0), iter_per_loop->GetOpDesc()->GetOutputDesc(0)}; std::vector output_desc_list; NodePtr stream_switch = InsertOp(compute_graph, STREAMSWITCH, switch_name, input_desc_list, output_desc_list); if (stream_switch == nullptr) { - GELOGE(FAILED, "InsertStreamSwitchOp failed, name:%s.", switch_name.c_str()); + GELOGE(FAILED, "[Insert][StreamSwitchOp] failed, name:%s.", switch_name.c_str()); return nullptr; } @@ -183,7 +185,9 @@ NodePtr FlowCtrlPass::InsertStreamSwitchOp(ComputeGraphPtr &compute_graph, const REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(index:0) and op:%s(%s)(index:0) failed", loop_cond->GetName().c_str(), loop_cond->GetType().c_str(), stream_switch->GetName().c_str(), stream_switch->GetType().c_str()); - GELOGE(FAILED, "Add loop_cond_node to switch_node:%s edge failed, ret = %u.", switch_name.c_str(), add_ret); + GELOGE(FAILED, "[Add][Edge] between op:%s(%s)(index:0) and op:%s(%s)(index:0) failed", + loop_cond->GetName().c_str(), loop_cond->GetType().c_str(), + stream_switch->GetName().c_str(), stream_switch->GetType().c_str()); return nullptr; } @@ -193,17 +197,20 @@ NodePtr FlowCtrlPass::InsertStreamSwitchOp(ComputeGraphPtr &compute_graph, const REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(index:0) and op:%s(%s)(index:1) failed", iter_per_loop->GetName().c_str(), iter_per_loop->GetType().c_str(), stream_switch->GetName().c_str(), stream_switch->GetType().c_str()); - GELOGE(FAILED, "Add iter_per_loop_node to switch_node:%s edge failed, ret = %u.", switch_name.c_str(), add_ret); + GELOGE(FAILED, "[Add][Edge] between op:%s(%s)(index:0) and op:%s(%s)(index:1) failed", + iter_per_loop->GetName().c_str(), iter_per_loop->GetType().c_str(), + stream_switch->GetName().c_str(), stream_switch->GetType().c_str()); return nullptr; } // stream switch op need switch cond by attr. GE_IF_BOOL_EXEC(!AttrUtils::SetInt(stream_switch->GetOpDesc(), ATTR_NAME_STREAM_SWITCH_COND, - static_cast(RT_LESS)), - REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed", - ATTR_NAME_STREAM_SWITCH_COND.c_str(), + static_cast(RT_LESS)), + REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed", ATTR_NAME_STREAM_SWITCH_COND.c_str(), stream_switch->GetName().c_str(), stream_switch->GetType().c_str()); - DOMI_LOGE("set ATTR_NAME_STREAM_SWITCH_COND failed"); return nullptr); + DOMI_LOGE("[Set][Attr] %s to op:%s(%s) failed", ATTR_NAME_STREAM_SWITCH_COND.c_str(), + stream_switch->GetName().c_str(), stream_switch->GetType().c_str()); + return nullptr); return stream_switch; } @@ -211,7 +218,7 @@ NodePtr FlowCtrlPass::InsertStreamSwitchOp(ComputeGraphPtr &compute_graph, const NodePtr FlowCtrlPass::AddVariableNode(ComputeGraphPtr &compute_graph, const string &name) { GE_IF_BOOL_EXEC(compute_graph == nullptr, REPORT_INNER_ERROR("E19999", "Param compute_graph is nullptr, check invalid"); - DOMI_LOGE("compute_graph is nullptr"); + DOMI_LOGE("[Check][Param] compute_graph is nullptr"); return nullptr); NodePtr exist_node = compute_graph->FindNode(name); if (exist_node != nullptr) { @@ -221,15 +228,14 @@ NodePtr FlowCtrlPass::AddVariableNode(ComputeGraphPtr &compute_graph, const stri // fetch and set tensor desc GeTensorDesc tensor_desc; if (ge::VarManager::Instance(compute_graph->GetSessionID()) == nullptr) { - REPORT_INNER_ERROR("E19999", "Get VarManager by session_id:%lu failed", - compute_graph->GetSessionID()); + REPORT_INNER_ERROR("E19999", "Get VarManager by session_id:%lu failed", compute_graph->GetSessionID()); return nullptr; } Status ret = ge::VarManager::Instance(compute_graph->GetSessionID())->GetCurVarDesc(name, tensor_desc); if (ret != SUCCESS) { REPORT_INNER_ERROR("E19999", "Get var tensor from VarManager by name:%s failed, session_id:%lu", name.c_str(), compute_graph->GetSessionID()); - GELOGE(FAILED, "Get var desc fail, name:%s", name.c_str()); + GELOGE(FAILED, "[Get][CurVarDesc] failed, name:%s, session_id:%lu", name.c_str(), compute_graph->GetSessionID()); return nullptr; } std::vector input_desc_list; @@ -263,7 +269,7 @@ Status FlowCtrlPass::AddGlobalStepVariableNode(ComputeGraphPtr &compute_graph) { NodePtr global_step = InsertOp(compute_graph, VARIABLE, NODE_NAME_GLOBAL_STEP, input_desc_list, output_desc_list); if (global_step == nullptr) { - GELOGE(FAILED, "Add global_step node failed, global_step is null."); + GELOGE(FAILED, "[Insert][Op] in graph:%s failed, global_step is null.", compute_graph->GetName().c_str()); return FAILED; } @@ -273,7 +279,9 @@ Status FlowCtrlPass::AddGlobalStepVariableNode(ComputeGraphPtr &compute_graph) { REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed", global_step->GetName().c_str(), global_step->GetType().c_str(), output_node->GetName().c_str(), output_node->GetType().c_str()); - GELOGE(FAILED, "Add global_step to netoutput edge failed, add_ret=%u.", add_ret); + GELOGE(FAILED, "[Add][ControlEdge] between op:%s(%s) and op:%s(%s) failed", + global_step->GetName().c_str(), global_step->GetType().c_str(), + output_node->GetName().c_str(), output_node->GetType().c_str()); return FAILED; } GELOGD("Add global_step to netoutput edge in graph %u success", compute_graph->GetGraphID()); @@ -284,9 +292,9 @@ NodePtr FlowCtrlPass::InsertAssignOp(ge::ComputeGraphPtr &compute_graph, const s const string &node_name, const NodePtr &ref_node, const NodePtr &value_node) { GE_IF_BOOL_EXEC(ref_node == nullptr || value_node == nullptr || ref_node->GetOpDesc() == nullptr || value_node->GetOpDesc() == nullptr, - REPORT_INNER_ERROR("E19999", "Param ref_node or value_node or their op_desc has nullptr, " + REPORT_INNER_ERROR("E19999", "Param ref_node or value_node or their op_desc is nullptr, " "check invalid"); - GELOGE(FAILED, "ref node or value node is null"); + GELOGE(FAILED, "[Check][Param] Param ref_node or value_node or their op_desc is nullptr"); return nullptr); GeTensorDesc ref_tensor_desc = ref_node->GetOpDesc()->GetOutputDesc(0); GeTensorDesc val_tensor_desc = value_node->GetOpDesc()->GetOutputDesc(0); @@ -294,7 +302,7 @@ NodePtr FlowCtrlPass::InsertAssignOp(ge::ComputeGraphPtr &compute_graph, const s std::vector output_desc_list = {ref_tensor_desc}; NodePtr assign_node = InsertOp(compute_graph, node_type, node_name, input_desc_list, output_desc_list); if (assign_node == nullptr) { - GELOGE(FAILED, "Insert node %s(%s) failed.", node_name.c_str(), node_type.c_str()); + GELOGE(FAILED, "[Insert][node] %s(%s) failed.", node_name.c_str(), node_type.c_str()); return nullptr; } // assign node input 0 = ref_node @@ -303,7 +311,9 @@ NodePtr FlowCtrlPass::InsertAssignOp(ge::ComputeGraphPtr &compute_graph, const s REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(index:0) and op:%s(%s)(index:0) failed", ref_node->GetName().c_str(), ref_node->GetType().c_str(), assign_node->GetName().c_str(), assign_node->GetType().c_str()); - GELOGE(FAILED, "Add ref_node to %s edge failed, add_ret=%u.", node_name.c_str(), add_ret); + GELOGE(FAILED, "[Add][Edge] between op:%s(%s)(index:0) and op:%s(%s)(index:0) failed", + ref_node->GetName().c_str(), ref_node->GetType().c_str(), + assign_node->GetName().c_str(), assign_node->GetType().c_str()); return nullptr; } // assign input 1 = value_node @@ -312,7 +322,9 @@ NodePtr FlowCtrlPass::InsertAssignOp(ge::ComputeGraphPtr &compute_graph, const s REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(index:0) and op:%s(%s)(index:1) failed", value_node->GetName().c_str(), value_node->GetType().c_str(), assign_node->GetName().c_str(), assign_node->GetType().c_str()); - GELOGE(FAILED, "Add value_node to %s edge failed, add_ret=%u.", node_name.c_str(), add_ret); + GELOGE(FAILED, "[Add][Edge] between op:%s(%s)(index:0) and op:%s(%s)(index:1) failed", + value_node->GetName().c_str(), value_node->GetType().c_str(), + assign_node->GetName().c_str(), assign_node->GetType().c_str()); return nullptr; } (void)ge::AttrUtils::SetBool(assign_node->GetOpDesc(), ATTR_NEED_COMPILE, true); @@ -335,7 +347,7 @@ Status FlowCtrlPass::CreateIterCtrlTrueBranch(ComputeGraphPtr &compute_graph, co assign_add_node_in_fpbp_loop_ = InsertAssignOp(compute_graph, ASSIGNADD, NODE_NAME_FLOWCTRL_LOOP_ASSIGNADD, loop_cond_node, loop_inc_node); if (assign_add_node_in_fpbp_loop_ == nullptr || switch_node == nullptr) { - GELOGE(PARAM_INVALID, "assign add node or switch node is null"); + GELOGE(PARAM_INVALID, "[Check][Param] assign add node or switch node is null"); return FAILED; } @@ -346,7 +358,9 @@ Status FlowCtrlPass::CreateIterCtrlTrueBranch(ComputeGraphPtr &compute_graph, co REPORT_CALL_ERROR("E19999", "Set stream_label:%s to op:%s(%s) failed", active_name.c_str(), assign_add_node_in_fpbp_loop_->GetName().c_str(), assign_add_node_in_fpbp_loop_->GetType().c_str()); - GELOGE(status, "Set stream label failed."); + GELOGE(status, "[Set][StreamLabel] %s to op:%s(%s) failed", + active_name.c_str(), assign_add_node_in_fpbp_loop_->GetName().c_str(), + assign_add_node_in_fpbp_loop_->GetType().c_str()); return status; } @@ -355,28 +369,31 @@ Status FlowCtrlPass::CreateIterCtrlTrueBranch(ComputeGraphPtr &compute_graph, co if (status != ge::SUCCESS) { REPORT_CALL_ERROR("E19999", "Set active label list:%s to op:%s(%s) failed", active_name.c_str(), switch_node->GetName().c_str(), switch_node->GetType().c_str()); - GELOGE(status, "set active_label_list failed."); + GELOGE(status, "[Set][ActiveLabelList] %s to op:%s(%s) failed", + active_name.c_str(), switch_node->GetName().c_str(), switch_node->GetType().c_str()); return status; } // 2. Insert active node NodePtr active_node = InsertOp(compute_graph, STREAMACTIVE, active_name, {}, {}); if (active_node == nullptr) { - GELOGE(FAILED, "Insert stream active node:%s for IterCtrlTrueStream failed.", active_name.c_str()); + GELOGE(FAILED, "[Insert][StreamActiveNode] %s for IterCtrlTrueStream failed.", active_name.c_str()); return FAILED; } status = SetStreamLabel(active_node, active_name); if (status != ge::SUCCESS) { REPORT_CALL_ERROR("E19999", "Set stream_label:%s to op:%s(%s) failed", active_name.c_str(), active_node->GetName().c_str(), active_node->GetType().c_str()); - GELOGE(status, "Set stream label failed."); + GELOGE(status, "[Set][StreamLabel] %s to op:%s(%s) failed", + active_name.c_str(), active_node->GetName().c_str(), active_node->GetType().c_str()); return status; } GE_IF_BOOL_EXEC(!AttrUtils::SetBool(active_node->GetOpDesc(), ATTR_NAME_IS_LOOP_ACTIVE, true), REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed", ATTR_NAME_IS_LOOP_ACTIVE.c_str(), active_node->GetName().c_str(), active_node->GetType().c_str()); - DOMI_LOGE("set ATTR_NAME_IS_LOOP_ACTIVE failed"); + DOMI_LOGE("[Set][Attr] %s to op:%s(%s) failed", ATTR_NAME_IS_LOOP_ACTIVE.c_str(), + active_node->GetName().c_str(), active_node->GetType().c_str()); return FAILED); // add ctrl edges @@ -387,7 +404,9 @@ Status FlowCtrlPass::CreateIterCtrlTrueBranch(ComputeGraphPtr &compute_graph, co switch_node->GetName().c_str(), switch_node->GetType().c_str(), assign_add_node_in_fpbp_loop_->GetName().c_str(), assign_add_node_in_fpbp_loop_->GetType().c_str()); - GELOGE(FAILED, "Add switch_node to assign_add_node ctrl edge failed, add_ret=%u.", add_ret); + GELOGE(FAILED, "[Add][ControlEdge] between op:%s(%s) and op:%s(%s) failed", + switch_node->GetName().c_str(), switch_node->GetType().c_str(), + assign_add_node_in_fpbp_loop_->GetName().c_str(), assign_add_node_in_fpbp_loop_->GetType().c_str()); return FAILED; } @@ -398,7 +417,9 @@ Status FlowCtrlPass::CreateIterCtrlTrueBranch(ComputeGraphPtr &compute_graph, co assign_add_node_in_fpbp_loop_->GetName().c_str(), assign_add_node_in_fpbp_loop_->GetType().c_str(), active_node->GetName().c_str(), active_node->GetType().c_str()); - GELOGE(FAILED, "Add assign_add_node to active_node ctrl edge failed, add_ret=%u.", add_ret); + GELOGE(FAILED, "[Add][ControlEdge] between op:%s(%s) and op:%s(%s) failed", + assign_add_node_in_fpbp_loop_->GetName().c_str(), assign_add_node_in_fpbp_loop_->GetType().c_str(), + active_node->GetName().c_str(), active_node->GetType().c_str()); return FAILED; } @@ -421,7 +442,7 @@ Status FlowCtrlPass::CreateIterCtrlFalseBranch(ComputeGraphPtr &compute_graph, c NodePtr assign_node = InsertAssignOp(compute_graph, ASSIGN, NODE_NAME_FLOWCTRL_LOOP_ASSIGN, loop_cond_node, loop_reset_node); if (assign_node == nullptr || switch_node == nullptr) { - GELOGE(PARAM_INVALID, "assign_node or switch node is null."); + GELOGE(PARAM_INVALID, "[Check][Param] assign_node or switch node is null."); return FAILED; } @@ -429,7 +450,8 @@ Status FlowCtrlPass::CreateIterCtrlFalseBranch(ComputeGraphPtr &compute_graph, c if (status != ge::SUCCESS) { REPORT_CALL_ERROR("E19999", "Set stream_label:%s to op:%s(%s) failed", switch_node->GetName().c_str(), assign_node->GetName().c_str(), assign_node->GetType().c_str()); - GELOGE(status, "Set stream label failed."); + GELOGE(status, "[Set][StreamLabel] %s to op:%s(%s) failed", + switch_node->GetName().c_str(), assign_node->GetName().c_str(), assign_node->GetType().c_str()); return status; } @@ -438,7 +460,9 @@ Status FlowCtrlPass::CreateIterCtrlFalseBranch(ComputeGraphPtr &compute_graph, c REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed", switch_node->GetName().c_str(), switch_node->GetType().c_str(), assign_node->GetName().c_str(), assign_node->GetType().c_str()); - GELOGE(FAILED, "Add switch_node to assign_node ctrl edge failed, add_ret=%u.", add_ret); + GELOGE(FAILED, "[Add][ControlEdge] between op:%s(%s) and op:%s(%s) failed", + switch_node->GetName().c_str(), switch_node->GetType().c_str(), + assign_node->GetName().c_str(), assign_node->GetType().c_str()); return FAILED; } @@ -448,25 +472,28 @@ Status FlowCtrlPass::CreateIterCtrlFalseBranch(ComputeGraphPtr &compute_graph, c string active_name = switch_node->GetName() + "_StreamExitActive"; NodePtr active_node = InsertOp(compute_graph, STREAMACTIVE, active_name, {}, {}); if (active_node == nullptr) { - GELOGE(FAILED, "Insert stream active node:%s for IterCtrlTrueStream failed.", active_name.c_str()); + GELOGE(FAILED, "[Insert][StreamActiveNode] %s for IterCtrlTrueStream failed.", active_name.c_str()); return FAILED; } status = SetStreamLabel(active_node, switch_node->GetName()); if (status != ge::SUCCESS) { - REPORT_CALL_ERROR("E19999", "Set stream_label:%s to op:%s(%s) failed", - switch_node->GetName().c_str(), active_node->GetName().c_str(), active_node->GetType().c_str()); - GELOGE(status, "Set stream label failed."); + REPORT_CALL_ERROR("E19999", "Set stream_label:%s to op:%s(%s) failed", switch_node->GetName().c_str(), + active_node->GetName().c_str(), active_node->GetType().c_str()); + GELOGE(status, "[Set][StreamLabel] %s to op:%s(%s) failed", switch_node->GetName().c_str(), + active_node->GetName().c_str(), active_node->GetType().c_str()); return status; } GE_CHK_STATUS_RET(SetSwitchBranchNodeLabel(active_node, switch_node->GetName()), - "set switch branch node label failed."); + "[Set][SwitchBranchNodeLabel] %s to op:%s(%s) failed", switch_node->GetName().c_str(), + active_node->GetName().c_str(), active_node->GetType().c_str()); string model_exit_name = switch_node->GetName() + "_ModelExit"; status = SetActiveLabelList(active_node, { model_exit_name }); if (status != ge::SUCCESS) { REPORT_CALL_ERROR("E19999", "Set active label list:%s to op:%s(%s) failed", model_exit_name.c_str(), active_node->GetName().c_str(), active_node->GetType().c_str()); - GELOGE(status, "set active_label_list failed."); + GELOGE(status, "[Set][ActiveLabelList] %s to op:%s(%s) failed", + model_exit_name.c_str(), active_node->GetName().c_str(), active_node->GetType().c_str()); return status; } @@ -475,14 +502,16 @@ Status FlowCtrlPass::CreateIterCtrlFalseBranch(ComputeGraphPtr &compute_graph, c REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed", assign_node->GetName().c_str(), assign_node->GetType().c_str(), active_node->GetName().c_str(), active_node->GetType().c_str()); - GELOGE(FAILED, "Add assign_node to active_node ctrl edge failed, add_ret=%u.", add_ret); + GELOGE(FAILED, "[Add][ControlEdge] between op:%s(%s) and op:%s(%s) failed", + assign_node->GetName().c_str(), assign_node->GetType().c_str(), + active_node->GetName().c_str(), active_node->GetType().c_str()); return FAILED; } // 3. Insert model exit node and add ctrl edge NodePtr model_exit_node = InsertOp(compute_graph, MODELEXIT, model_exit_name, {}, {}); if (model_exit_node == nullptr) { - GELOGE(FAILED, "Insert model_exit node:%s for IterCtrlTrueStream failed.", model_exit_name.c_str()); + GELOGE(FAILED, "[Insert][ModelExitNode] %s for IterCtrlTrueStream failed.", model_exit_name.c_str()); return FAILED; } status = SetStreamLabel(model_exit_node, model_exit_name); @@ -490,7 +519,8 @@ Status FlowCtrlPass::CreateIterCtrlFalseBranch(ComputeGraphPtr &compute_graph, c REPORT_CALL_ERROR("E19999", "Set stream_label:%s to op:%s(%s) failed", model_exit_name.c_str(), model_exit_node->GetName().c_str(), model_exit_node->GetType().c_str()); - GELOGE(status, "Set stream label failed."); + GELOGE(status, "[Set][StreamLabel] %s to op:%s(%s) failed", model_exit_name.c_str(), + model_exit_node->GetName().c_str(), model_exit_node->GetType().c_str()); return status; } @@ -499,7 +529,9 @@ Status FlowCtrlPass::CreateIterCtrlFalseBranch(ComputeGraphPtr &compute_graph, c REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed", active_node->GetName().c_str(), assign_node->GetType().c_str(), model_exit_node->GetName().c_str(), model_exit_node->GetType().c_str()); - GELOGE(FAILED, "Add active_node to model_exit_node ctrl edge failed, add_ret=%u.", add_ret); + GELOGE(FAILED, "[Add][ControlEdge] between op:%s(%s) and op:%s(%s) failed", + active_node->GetName().c_str(), assign_node->GetType().c_str(), + model_exit_node->GetName().c_str(), model_exit_node->GetType().c_str()); return FAILED; } } @@ -509,28 +541,28 @@ Status FlowCtrlPass::CreateIterCtrlFalseBranch(ComputeGraphPtr &compute_graph, c } Status FlowCtrlPass::AddFpBpIteratorCtrl(ComputeGraphPtr &compute_graph, NodePtr &pre_node) { - GE_IF_BOOL_EXEC(pre_node == nullptr, DOMI_LOGE("pre_node is nullptr."); return FAILED); + GE_IF_BOOL_EXEC(pre_node == nullptr, DOMI_LOGE("[Check][Param] pre_node is nullptr."); return FAILED); string pre_node_name = pre_node->GetName(); GELOGI("Add FpBp Iterator ctrl, pre node:%s.", pre_node_name.c_str()); // 1. Get or add variables NodePtr loop_cond_node = AddVariableNode(compute_graph, NODE_NAME_FLOWCTRL_LOOP_COND); if (loop_cond_node == nullptr) { - GELOGE(FAILED, "Add variable:%s failed.", NODE_NAME_FLOWCTRL_LOOP_COND.c_str()); + GELOGE(FAILED, "[Add][Variable] %s failed.", NODE_NAME_FLOWCTRL_LOOP_COND.c_str()); return FAILED; } NodePtr loop_inc_node = AddVariableNode(compute_graph, NODE_NAME_FLOWCTRL_LOOP_INCREMENT); if (loop_inc_node == nullptr) { - GELOGE(FAILED, "Add variable:%s failed.", NODE_NAME_FLOWCTRL_LOOP_INCREMENT.c_str()); + GELOGE(FAILED, "[Add][Variable] %s failed.", NODE_NAME_FLOWCTRL_LOOP_INCREMENT.c_str()); return FAILED; } NodePtr loop_reset_node = AddVariableNode(compute_graph, NODE_NAME_FLOWCTRL_LOOP_RESETVALUE); if (loop_reset_node == nullptr) { - GELOGE(FAILED, "Add variable:%s failed.", NODE_NAME_FLOWCTRL_LOOP_RESETVALUE.c_str()); + GELOGE(FAILED, "[Add][Variable] %s failed.", NODE_NAME_FLOWCTRL_LOOP_RESETVALUE.c_str()); return FAILED; } NodePtr iter_per_loop_node = AddVariableNode(compute_graph, NODE_NAME_FLOWCTRL_LOOP_PER_ITER); if (iter_per_loop_node == nullptr) { - GELOGE(FAILED, "Add variable:%s failed.", NODE_NAME_FLOWCTRL_LOOP_PER_ITER.c_str()); + GELOGE(FAILED, "[Add][Variable] %s failed.", NODE_NAME_FLOWCTRL_LOOP_PER_ITER.c_str()); return FAILED; } @@ -538,14 +570,15 @@ Status FlowCtrlPass::AddFpBpIteratorCtrl(ComputeGraphPtr &compute_graph, NodePtr string switch_name = pre_node_name + "_" + NODE_NAME_STREAM_SWITCH; NodePtr switch_node = InsertStreamSwitchOp(compute_graph, switch_name, loop_cond_node, iter_per_loop_node); if (switch_node == nullptr) { - GELOGE(FAILED, "InsertStreamSwitchOp:%s failed.", switch_name.c_str()); + GELOGE(FAILED, "[Insert][StreamSwitchOp] %s failed.", switch_name.c_str()); return FAILED; } auto status = SetStreamLabel(switch_node, switch_name); if (status != ge::SUCCESS) { REPORT_CALL_ERROR("E19999", "Set stream label:%s to op:%s(%s) failed", switch_name.c_str(), switch_node->GetName().c_str(), switch_node->GetType().c_str()); - GELOGE(status, "set stream label failed."); + GELOGE(status, "[Set][StreamLabel] %s to op:%s(%s) failed", + switch_name.c_str(), switch_node->GetName().c_str(), switch_node->GetType().c_str()); return status; } @@ -554,15 +587,16 @@ Status FlowCtrlPass::AddFpBpIteratorCtrl(ComputeGraphPtr &compute_graph, NodePtr REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed", pre_node->GetName().c_str(), pre_node->GetType().c_str(), switch_node->GetName().c_str(), switch_node->GetType().c_str()); - GELOGE(FAILED, "Add pre node:%s to switch_node:%s ctrl edge failed, ret = %u.", pre_node_name.c_str(), - switch_name.c_str(), add_ret); + GELOGE(FAILED, "[Add][ControlEdge] between op:%s(%s) and op:%s(%s) failed", + pre_node->GetName().c_str(), pre_node->GetType().c_str(), + switch_node->GetName().c_str(), switch_node->GetType().c_str()); return FAILED; } // 3. Create switch false branch: return results and reset the loopCond Status ret = CreateIterCtrlFalseBranch(compute_graph, loop_cond_node, loop_reset_node, switch_node); if (ret != SUCCESS) { - GELOGE(ret, "CreateIterCtrlFalseBranch fail, pre node:%s.", pre_node_name.c_str()); + GELOGE(ret, "[Create][IterCtrlFalseBranch] fail, pre node:%s.", pre_node_name.c_str()); return ret; } @@ -570,7 +604,7 @@ Status FlowCtrlPass::AddFpBpIteratorCtrl(ComputeGraphPtr &compute_graph, NodePtr // active train streams and increase the loopCond ret = CreateIterCtrlTrueBranch(compute_graph, loop_cond_node, loop_inc_node, switch_node); if (ret != SUCCESS) { - GELOGE(ret, "CreateIterCtrlTrueBranch fail, pre node:%s.", pre_node_name.c_str()); + GELOGE(ret, "[Create][IterCtrlTrueBranch] fail, pre node:%s.", pre_node_name.c_str()); return ret; } return SUCCESS; @@ -594,15 +628,15 @@ Status FlowCtrlPass::AddSpecialNodeIteratorCtrl(ComputeGraphPtr &compute_graph, * itersPerLoop loopCond */ GE_IF_BOOL_EXEC(loop_after_node == nullptr || compute_graph == nullptr, - REPORT_INNER_ERROR("E19999", "Param loop_after_node or compute_graph is nullptr, " - "check invalid"); - DOMI_LOGE("loop after node or compute graph is null."); + REPORT_INNER_ERROR("E19999", "Param loop_after_node or compute_graph is nullptr, check invalid"); + DOMI_LOGE("[Check][Param] loop after node or compute graph is null."); return FAILED); InDataAnchorPtr in_anchor = loop_after_node->GetInDataAnchor(0); if (in_anchor == nullptr || in_anchor->GetPeerOutAnchor() == nullptr) { REPORT_INNER_ERROR("E19999", "Param loop_after_node:%s(%s) no in data node, check invalid", loop_after_node->GetName().c_str(), loop_after_node->GetType().c_str()); - GELOGE(FAILED, "Find %s in data anchor failed.", loop_after_node->GetName().c_str()); + GELOGE(FAILED, "[Check][Param] Param loop_after_node:%s(%s) no in data node.", + loop_after_node->GetName().c_str(), loop_after_node->GetType().c_str()); return FAILED; } NodePtr loop_pre_node = in_anchor->GetPeerOutAnchor()->GetOwnerNode(); @@ -612,28 +646,30 @@ Status FlowCtrlPass::AddSpecialNodeIteratorCtrl(ComputeGraphPtr &compute_graph, if (loop_cond_node == nullptr) { REPORT_INNER_ERROR("E19999", "Node:%s not found in graph:%s, check invalid", NODE_NAME_FLOWCTRL_LOOP_COND.c_str(), compute_graph->GetName().c_str()); - GELOGE(FAILED, "Find node :%s failed.", NODE_NAME_FLOWCTRL_LOOP_COND.c_str()); + GELOGE(FAILED, "[Check][Param] Node:%s not found in graph:%s.", + NODE_NAME_FLOWCTRL_LOOP_COND.c_str(), compute_graph->GetName().c_str()); return FAILED; } NodePtr iter_per_loop_node = compute_graph->FindNode(NODE_NAME_FLOWCTRL_LOOP_PER_ITER); if (iter_per_loop_node == nullptr) { REPORT_INNER_ERROR("E19999", "Node:%s not found in graph:%s, check invalid", NODE_NAME_FLOWCTRL_LOOP_PER_ITER.c_str(), compute_graph->GetName().c_str()); - GELOGE(FAILED, "Find node :%s failed.", NODE_NAME_FLOWCTRL_LOOP_PER_ITER.c_str()); + GELOGE(FAILED, "[Check][Param] Node:%s not found in graph:%s.", + NODE_NAME_FLOWCTRL_LOOP_PER_ITER.c_str(), compute_graph->GetName().c_str()); return FAILED; } // 2. Add StreamSwitch and edges to switch_node. GE_IF_BOOL_EXEC(loop_pre_node == nullptr, - REPORT_INNER_ERROR("E19999", "Param loop_after_node:%s(%s) no in data node, " - "check invalid", loop_after_node->GetName().c_str(), - loop_after_node->GetType().c_str()); - DOMI_LOGE("loop pre node is null."); + REPORT_INNER_ERROR("E19999", "Param loop_after_node:%s(%s) no in data node, check invalid", + loop_after_node->GetName().c_str(), loop_after_node->GetType().c_str()); + DOMI_LOGE("[Check][Param] Param loop_after_node:%s(%s) no in data node", + loop_after_node->GetName().c_str(), loop_after_node->GetType().c_str()); return FAILED); string switch_name = loop_pre_node->GetName() + "_" + NODE_NAME_STREAM_SWITCH; NodePtr switch_node = InsertStreamSwitchOp(compute_graph, switch_name, loop_cond_node, iter_per_loop_node); if (switch_node == nullptr) { - GELOGE(FAILED, "InsertStreamSwitchOp:%s failed.", switch_name.c_str()); + GELOGE(FAILED, "[Insert][StreamSwitchOp] %s failed.", switch_name.c_str()); return FAILED; } @@ -641,7 +677,8 @@ Status FlowCtrlPass::AddSpecialNodeIteratorCtrl(ComputeGraphPtr &compute_graph, if (status != ge::SUCCESS) { REPORT_CALL_ERROR("E19999", "Set stream label:%s to op:%s(%s) failed", switch_name.c_str(), switch_node->GetName().c_str(), switch_node->GetType().c_str()); - GELOGE(status, "set stream label failed."); + GELOGE(status, "[Set][StreamLabel] %s to op:%s(%s) failed", + switch_name.c_str(), switch_node->GetName().c_str(), switch_node->GetType().c_str()); return status; } @@ -650,8 +687,9 @@ Status FlowCtrlPass::AddSpecialNodeIteratorCtrl(ComputeGraphPtr &compute_graph, REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed", loop_pre_node->GetName().c_str(), loop_pre_node->GetType().c_str(), switch_node->GetName().c_str(), switch_node->GetType().c_str()); - GELOGE(FAILED, "Add loop_pre_node:%s to switch_node:%s ctrl edge failed, ret = %u.", - loop_pre_node->GetName().c_str(), switch_name.c_str(), add_ret); + GELOGE(FAILED, "[Add][ControlEdge] between op:%s(%s) and op:%s(%s) failed", + loop_pre_node->GetName().c_str(), loop_pre_node->GetType().c_str(), + switch_node->GetName().c_str(), switch_node->GetType().c_str()); return FAILED; } add_ret = GraphUtils::AddEdge(loop_after_node->GetOutControlAnchor(), switch_node->GetInControlAnchor()); @@ -659,8 +697,9 @@ Status FlowCtrlPass::AddSpecialNodeIteratorCtrl(ComputeGraphPtr &compute_graph, REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed", loop_after_node->GetName().c_str(), loop_after_node->GetType().c_str(), switch_node->GetName().c_str(), switch_node->GetType().c_str()); - GELOGE(FAILED, "Add node:%s to switch_node:%s ctrl edge failed, ret = %u.", loop_after_node->GetName().c_str(), - switch_name.c_str(), add_ret); + GELOGE(FAILED, "[Add][ControlEdge] between op:%s(%s) and op:%s(%s) failed", + loop_after_node->GetName().c_str(), loop_after_node->GetType().c_str(), + switch_node->GetName().c_str(), switch_node->GetType().c_str()); return FAILED; } @@ -668,7 +707,7 @@ Status FlowCtrlPass::AddSpecialNodeIteratorCtrl(ComputeGraphPtr &compute_graph, string active_name = switch_name + "_StreamActive"; NodePtr active_node = InsertOp(compute_graph, STREAMACTIVE, active_name, {}, {}); if (active_node == nullptr) { - GELOGE(FAILED, "Insert stream active node:%s for SpecialNodeIteratorCtrl failed.", active_name.c_str()); + GELOGE(FAILED, "[Insert][StreamActiveNode] %s for SpecialNodeIteratorCtrl failed.", active_name.c_str()); return FAILED; } @@ -676,15 +715,16 @@ Status FlowCtrlPass::AddSpecialNodeIteratorCtrl(ComputeGraphPtr &compute_graph, if (status != ge::SUCCESS) { REPORT_CALL_ERROR("E19999", "Set stream label:%s to op:%s(%s) failed", active_name.c_str(), active_node->GetName().c_str(), active_node->GetType().c_str()); - GELOGE(status, "set stream label failed."); + GELOGE(status, "[Set][StreamLabel] %s to op:%s(%s) failed", + active_name.c_str(), active_node->GetName().c_str(), active_node->GetType().c_str()); return status; } GE_IF_BOOL_EXEC(!AttrUtils::SetBool(active_node->GetOpDesc(), ATTR_NAME_IS_LOOP_ACTIVE, true), - REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed", - ATTR_NAME_IS_LOOP_ACTIVE.c_str(), + REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed", ATTR_NAME_IS_LOOP_ACTIVE.c_str(), active_node->GetName().c_str(), active_node->GetType().c_str()); - DOMI_LOGE("set ATTR_NAME_IS_LOOP_ACTIVE failed"); + DOMI_LOGE("[Set][Attr] %s to op:%s(%s) failed", ATTR_NAME_IS_LOOP_ACTIVE.c_str(), + active_node->GetName().c_str(), active_node->GetType().c_str()); return FAILED); add_ret = GraphUtils::AddEdge(switch_node->GetOutControlAnchor(), active_node->GetInControlAnchor()); @@ -692,8 +732,9 @@ Status FlowCtrlPass::AddSpecialNodeIteratorCtrl(ComputeGraphPtr &compute_graph, REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed", switch_node->GetName().c_str(), switch_node->GetType().c_str(), active_node->GetName().c_str(), active_node->GetType().c_str()); - GELOGE(FAILED, "Add switch_node:%s to active_node:%s ctrl edge failed, ret = %u.", switch_name.c_str(), - active_name.c_str(), add_ret); + GELOGE(FAILED, "[Add][ControlEdge] between op:%s(%s) and op:%s(%s) failed", + switch_node->GetName().c_str(), switch_node->GetType().c_str(), + active_node->GetName().c_str(), active_node->GetType().c_str()); return FAILED; } @@ -702,15 +743,17 @@ Status FlowCtrlPass::AddSpecialNodeIteratorCtrl(ComputeGraphPtr &compute_graph, if (status != ge::SUCCESS) { REPORT_CALL_ERROR("E19999", "Set active label list:%s to op:%s(%s) failed", active_name.c_str(), switch_node->GetName().c_str(), switch_node->GetType().c_str()); - GELOGE(status, "set active_label_list failed."); + GELOGE(status, "[Set][ActiveLabelList] %s to op:%s(%s) failed", + active_name.c_str(), switch_node->GetName().c_str(), switch_node->GetType().c_str()); return status; } // used for stream assign to find active stream status = SetActiveLabelList(active_node, { loop_pre_node->GetName() }); if (status != ge::SUCCESS) { - REPORT_CALL_ERROR("E19999", "Set active label list:%s to op:%s(%s) failed", - loop_pre_node->GetName().c_str(), active_node->GetName().c_str(), active_node->GetType().c_str()); - GELOGE(status, "set active_label_list failed."); + REPORT_CALL_ERROR("E19999", "Set active label list:%s to op:%s(%s) failed", loop_pre_node->GetName().c_str(), + active_node->GetName().c_str(), active_node->GetType().c_str()); + GELOGE(status, "[Set][ActiveLabelList] %s to op:%s(%s) failed", + loop_pre_node->GetName().c_str(), active_node->GetName().c_str(), active_node->GetType().c_str()); return status; } active_nodes_in_iter_loop_.push_back(active_node); diff --git a/ge/graph/passes/folding_pass.cc b/ge/graph/passes/folding_pass.cc index d4558ac7..c0a0f2a2 100755 --- a/ge/graph/passes/folding_pass.cc +++ b/ge/graph/passes/folding_pass.cc @@ -36,7 +36,7 @@ namespace folding_pass { shared_ptr GetKernelByType(const NodePtr &node) { if (node == nullptr) { REPORT_INNER_ERROR("E19999", "Param node is nullptr, check invalid"); - GELOGE(FAILED, "parameter is null."); + GELOGE(FAILED, "[Check][Param] parameter node is nullptr."); return nullptr; } KernelFactory &factory = KernelFactory::Instance(); @@ -84,7 +84,7 @@ NodePtr AddConstNodeToGraph(GeTensorPtr &tensor, ComputeGraphPtr &graph) { auto const_desc = OpDescUtils::CreateConstOp(tensor); if (const_desc == nullptr) { REPORT_CALL_ERROR("E19999", "Create Const op failed"); - GELOGE(OUT_OF_MEMORY, "Failed to get const desc from tensor"); + GELOGE(OUT_OF_MEMORY, "[Create][ConstOp] failed"); return nullptr; } @@ -96,14 +96,14 @@ NodePtr AddConstNodeToGraph(GeTensorPtr &tensor, ComputeGraphPtr &graph) { NodePtr AddIdentityNodeToGraph(const std::string &name, const GeTensorDesc &tensor, ComputeGraphPtr &graph) { if (graph == nullptr) { REPORT_INNER_ERROR("E19999", "Param graph is nullptr, check invalid"); - GELOGE(INTERNAL_ERROR, "Compute graph ptr is null in creating identity node."); + GELOGE(INTERNAL_ERROR, "[Check][Param] Compute graph ptr is null in creating identity node."); return nullptr; } OpDescPtr desc = MakeShared("", ""); if (desc == nullptr) { REPORT_CALL_ERROR("E19999", "New OpDesc failed"); - GELOGE(MEMALLOC_FAILED, "Failed to create op desc."); + GELOGE(MEMALLOC_FAILED, "[New][OpDesc] failed."); return nullptr; } @@ -114,7 +114,8 @@ NodePtr AddIdentityNodeToGraph(const std::string &name, const GeTensorDesc &tens if ((ret != GRAPH_SUCCESS) || (ret2 != GRAPH_SUCCESS)) { REPORT_CALL_ERROR("E19999", "Add input or output desc to op:%s(%s) failed", desc->GetName().c_str(), desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to add input/output desc in creating Identity."); + GELOGE(INTERNAL_ERROR, "[Add][GeTensorDesc] to op:%s(%s) failed", + desc->GetName().c_str(), desc->GetType().c_str()); return nullptr; } @@ -146,9 +147,9 @@ Status FoldingPass::Folding(NodePtr &node, vector &outputs) { auto in_data_nodes = node->GetInDataNodes(); std::unordered_set in_data_nodes_set(in_data_nodes.begin(), in_data_nodes.end()); if (IsolateAndDeleteNode(node, {}) != SUCCESS) { - REPORT_INNER_ERROR("E19999", "Isolate and delete node:%s(%s) faild", + REPORT_INNER_ERROR("E19999", "Isolate and delete node:%s(%s) failed", node->GetName().c_str(), node->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to isolate and delete node %s, type %s.", + GELOGE(INTERNAL_ERROR, "[IsolateAndDelete][Node] %s(%s) failed.", node->GetName().c_str(), node->GetType().c_str()); return INTERNAL_ERROR; } @@ -160,9 +161,9 @@ Status FoldingPass::Folding(NodePtr &node, vector &outputs) { continue; } if (IsolateAndDeleteNode(pre_node, {}) != SUCCESS) { - REPORT_INNER_ERROR("E19999", "Isolate and delete node:%s(%s) faild", + REPORT_INNER_ERROR("E19999", "Isolate and delete node:%s(%s) failed", pre_node->GetName().c_str(), pre_node->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to isolate and delete in data node %s, type %s.", + GELOGE(INTERNAL_ERROR, "[IsolateAndDelete][Node] %s(%s) failed.", pre_node->GetName().c_str(), pre_node->GetType().c_str()); return INTERNAL_ERROR; } @@ -190,11 +191,10 @@ Status FoldingPass::DealWithInNodes(NodePtr &node) { GELOGI("The in_node name is %s, and node type is %s.", in_node->GetName().c_str(), in_node->GetType().c_str()); auto ret = in_node_anchor->Unlink(in_data_anchor); if (ret != SUCCESS) { - REPORT_CALL_ERROR("E19999", - "Op:%s(%s) out index:%d unlink from op:%s(%s) in index:%d failed", + REPORT_CALL_ERROR("E19999", "Op:%s(%s) out index:%d unlink from op:%s(%s) in index:%d failed", in_node->GetName().c_str(), in_node->GetType().c_str(), in_node_anchor->GetIdx(), node->GetName().c_str(), node->GetType().c_str(), in_data_anchor->GetIdx()); - GELOGE(INTERNAL_ERROR, "Failed to unlink anchor between const node %s to constant-folding-node %s, type %s.", + GELOGE(INTERNAL_ERROR, "[Unlink][Anchor] between const node:%s and constant-folding-node:%s(%s) failed.", in_node->GetName().c_str(), node->GetName().c_str(), node->GetType().c_str()); return INTERNAL_ERROR; } @@ -204,16 +204,18 @@ Status FoldingPass::DealWithInNodes(NodePtr &node) { auto identity = AddIdentityNodeToGraph(identity_name, node->GetOpDesc()->GetInputDesc(in_data_anchor->GetIdx()), graph); if (identity == nullptr) { - GELOGE(INTERNAL_ERROR, "Failed to add identity node to graph."); + GELOGE(INTERNAL_ERROR, "[Add][IdentityNode] %s to graph:%s failed.", + identity_name.c_str(), graph->GetName().c_str()); return INTERNAL_ERROR; } ret = GraphUtils::AddEdge(in_node_anchor, identity->GetInDataAnchor(0)); if (ret != GRAPH_SUCCESS) { - REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(index:%d) and op:%s(%s)(inde:0) failed", + REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(index:%d) and op:%s(%s)(index:0) failed", in_node->GetName().c_str(), in_node->GetType().c_str(), in_node_anchor->GetIdx(), identity->GetName().c_str(), identity->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to add edge, from node %s to node %s.", in_node->GetName().c_str(), - identity->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Add][Edge] between op:%s(%s)(index:%d) and op:%s(%s)(index:0) failed", + in_node->GetName().c_str(), in_node->GetType().c_str(), in_node_anchor->GetIdx(), + identity->GetName().c_str(), identity->GetType().c_str()); return INTERNAL_ERROR; } GELOGI("Create new identity node success."); @@ -222,8 +224,9 @@ Status FoldingPass::DealWithInNodes(NodePtr &node) { REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed", identity->GetName().c_str(), identity->GetType().c_str(), node->GetName().c_str(), node->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to add edge, from node %s to node %s.", in_node->GetName().c_str(), - node->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Add][ControlEdge] between op:%s(%s) and op:%s(%s) failed", + identity->GetName().c_str(), identity->GetType().c_str(), + node->GetName().c_str(), node->GetType().c_str()); return INTERNAL_ERROR; } } @@ -236,7 +239,7 @@ Status FoldingPass::AddConstNode(NodePtr &node, IndexsToAnchors indexes_to_ancho std::vector &v_weight) { if (node == nullptr) { REPORT_INNER_ERROR("E19999", "Param node is nullptr, check invalid"); - GELOGE(PARAM_INVALID, "node is null"); + GELOGE(PARAM_INVALID, "[Check][Param] node is nullptr"); return FAILED; } auto graph = node->GetOwnerComputeGraph(); @@ -245,24 +248,23 @@ Status FoldingPass::AddConstNode(NodePtr &node, IndexsToAnchors indexes_to_ancho if (index >= v_weight.size()) { REPORT_INNER_ERROR("E19999", "Index:%lu in param index_to_anchors >= param v_weight.size:%zu, " "check invalid", index, v_weight.size()); - GELOGE(INTERNAL_ERROR, - "Failed to constant fold on node %s type %s, " + GELOGE(INTERNAL_ERROR, "[Check][Param] Failed to constant fold on node %s type %s, " "the out nodes num %lu calculated is less than the node out anchor index %zu", node->GetName().c_str(), node->GetType().c_str(), v_weight.size(), index); return INTERNAL_ERROR; } GeTensorPtr weight = v_weight[index]; if (weight == nullptr) { - REPORT_INNER_ERROR("E19999", "Index:%lu in param v_weight is nullptr check invalid", - index); - GELOGE(INTERNAL_ERROR, "Failed to constant fold on node %s type %s, the %lust node calculated is null", + REPORT_INNER_ERROR("E19999", "Index:%lu in param v_weight is nullptr check invalid", index); + GELOGE(INTERNAL_ERROR, + "[Check][Param] Failed to constant fold on node %s type %s, the %lust node calculated is null", node->GetName().c_str(), node->GetType().c_str(), index); return INTERNAL_ERROR; } auto const_node = AddConstNodeToGraph(weight, graph); if (const_node == nullptr) { - GELOGE(INTERNAL_ERROR, "Failed to add dynamic const node, node name:%s, index:%zu.", + GELOGE(INTERNAL_ERROR, "[Add][ConstNode] To Graph failed, node name:%s, index:%zu.", node->GetName().c_str(), index); return INTERNAL_ERROR; } @@ -273,7 +275,8 @@ Status FoldingPass::AddConstNode(NodePtr &node, IndexsToAnchors indexes_to_ancho if (in_anchor == nullptr) { REPORT_INNER_ERROR("E19999", "Index:%lu in param index_to_anchors has nullptr member in_anchor, " "check invalid", index); - GELOGE(INTERNAL_ERROR, "In anchor is nullptr."); + GELOGE(INTERNAL_ERROR, + "[Check][Param] Index:%lu in param index_to_anchors has nullptr member in_anchor", index); return INTERNAL_ERROR; } auto ret = ConnectNodeToInAnchor(in_anchor, const_node, 0); @@ -287,7 +290,7 @@ Status FoldingPass::AddConstNode(NodePtr &node, IndexsToAnchors indexes_to_ancho REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed", node->GetName().c_str(), node->GetType().c_str(), const_node->GetName().c_str(), const_node->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to add control edge, from node %s to const node %s.", node->GetName().c_str(), + GELOGE(INTERNAL_ERROR, "[Add][ControlEdge] failed, from node %s to const node %s.", node->GetName().c_str(), const_node->GetName().c_str()); return INTERNAL_ERROR; } @@ -296,11 +299,10 @@ Status FoldingPass::AddConstNode(NodePtr &node, IndexsToAnchors indexes_to_ancho if (AttrUtils::GetStr(node->GetOpDesc(), ATTR_NAME_STREAM_LABEL, stream_label)) { GE_CHECK_NOTNULL(const_node->GetOpDesc()); if (!AttrUtils::SetStr(const_node->GetOpDesc(), ATTR_NAME_STREAM_LABEL, stream_label)) { - REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed", - ATTR_NAME_STREAM_LABEL.c_str(), + REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed", ATTR_NAME_STREAM_LABEL.c_str(), const_node->GetName().c_str(), const_node->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to set stream label on dynamic const node %s, with stream label:%s.", - const_node->GetName().c_str(), stream_label.c_str()); + GELOGE(INTERNAL_ERROR, "[Set][Attr] %s to op:%s(%s) failed", ATTR_NAME_STREAM_LABEL.c_str(), + const_node->GetName().c_str(), const_node->GetType().c_str()); return INTERNAL_ERROR; } } @@ -317,7 +319,7 @@ Status FoldingPass::RemoveNodeKeepingCtrlEdges(NodePtr &node) { if (ret != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Isolate node:%s(%s) in graph failed", node->GetName().c_str(), node->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to isolate the folding-node %s type %s", node->GetName().c_str(), + GELOGE(INTERNAL_ERROR, "[Isolate][Node] %s type %s failed", node->GetName().c_str(), node->GetType().c_str()); return INTERNAL_ERROR; } @@ -327,7 +329,8 @@ Status FoldingPass::RemoveNodeKeepingCtrlEdges(NodePtr &node) { if (ret != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Remove node:%s(%s) without relink in graph:%s failed", node->GetName().c_str(), node->GetType().c_str(), graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to remove node %s from graph", node->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Remove][Node] %s(%s) without relink in graph:%s failed", + node->GetName().c_str(), node->GetType().c_str(), graph->GetName().c_str()); return INTERNAL_ERROR; } AddNodeDeleted(node); @@ -338,7 +341,7 @@ Status FoldingPass::ConnectNodeToInAnchor(InDataAnchorPtr &in_anchor, NodePtr &n // the origin edge must be removed before add if (in_anchor == nullptr || node == nullptr) { REPORT_INNER_ERROR("E19999", "Param node or in_anchor is nullptr, check invalid"); - GELOGE(PARAM_INVALID, "in anchor or node is null"); + GELOGE(PARAM_INVALID, "[Check][Param] in anchor or node is null"); return PARAM_INVALID; } auto peer_out_anchor = in_anchor->GetPeerOutAnchor(); @@ -352,8 +355,7 @@ Status FoldingPass::ConnectNodeToInAnchor(InDataAnchorPtr &in_anchor, NodePtr &n if (new_out_anchor == nullptr) { REPORT_INNER_ERROR("E19999", "Param out index:%d data anchor of node:%s(%s) is nullptr, check invalid", node_index, node->GetName().c_str(), node->GetType().c_str()); - GELOGE(INTERNAL_ERROR, - "Failed to add node to in anchor," + GELOGE(INTERNAL_ERROR, "[Check][Param] Failed to add node to in anchor," " the index %d for node %s, type %s is invalid", node_index, node->GetName().c_str(), node->GetType().c_str()); return INTERNAL_ERROR; @@ -363,10 +365,10 @@ Status FoldingPass::ConnectNodeToInAnchor(InDataAnchorPtr &in_anchor, NodePtr &n node->GetName().c_str(), node->GetType().c_str(), node_index, in_anchor->GetOwnerNode()->GetName().c_str(), in_anchor->GetOwnerNode()->GetType().c_str(), in_anchor->GetIdx()); - GELOGE(INTERNAL_ERROR, - "Failed to add edge between anchors," - " new node %s, type %s", - node->GetName().c_str(), node->GetType().c_str()); + GELOGE(INTERNAL_ERROR, "[Add][Edge] between op:%s(%s)(index:%d) and op:%s(%s)(index:%d) failed", + node->GetName().c_str(), node->GetType().c_str(), node_index, + in_anchor->GetOwnerNode()->GetName().c_str(), in_anchor->GetOwnerNode()->GetType().c_str(), + in_anchor->GetIdx()); return INTERNAL_ERROR; } AddRePassNodesWithInOut(node); diff --git a/ge/graph/passes/for_pass.cc b/ge/graph/passes/for_pass.cc index 0b6377dc..7d09f370 100644 --- a/ge/graph/passes/for_pass.cc +++ b/ge/graph/passes/for_pass.cc @@ -56,17 +56,18 @@ Status ForPass::Run(NodePtr &node) { ForInfo for_info; GE_CHK_STATUS_RET(BuildForInfo(root_graph, node, for_info), - "Build ForInfo failed, node:%s.", node->GetName().c_str()); + "[Build][ForInfo] failed, node:%s.", node->GetName().c_str()); WhileInfo while_info; GE_CHK_STATUS_RET(TranWhileInfo(graph, for_info, while_info), - "Transfer WhileInfo from ForInfo failed, node:%s.", node->GetName().c_str()); + "[Transfer][WhileInfo] from ForInfo failed, node:%s.", node->GetName().c_str()); ComputeGraphPtr cond_graph = BuildCondGraph(while_info); if ((cond_graph == nullptr) || (root_graph->AddSubgraph(cond_graph) != GRAPH_SUCCESS)) { REPORT_CALL_ERROR("E19999", "Build cond graph failed or add cond subgraph to root_graph:%s failed", root_graph->GetName().c_str()); - GELOGE(FAILED, "Add while_cond_graph failed, node:%s.", node->GetName().c_str()); + GELOGE(FAILED, "[Check][Param] Build cond graph failed or add cond subgraph to root_graph:%s failed.", + root_graph->GetName().c_str()); return FAILED; } @@ -74,12 +75,13 @@ Status ForPass::Run(NodePtr &node) { if ((body_graph == nullptr) || (root_graph->AddSubgraph(body_graph) != GRAPH_SUCCESS)) { REPORT_CALL_ERROR("E19999", "Build body graph failed or add body subgraph to root_graph:%s failed", root_graph->GetName().c_str()); - GELOGE(FAILED, "Add while_body_graph failed, node:%s.", node->GetName().c_str()); + GELOGE(FAILED, "[Check][Param] Build body graph failed or add body subgraph to root_graph:%s failed", + root_graph->GetName().c_str()); return FAILED; } GE_CHK_STATUS_RET(UpdateForBodyInputMapping(while_info), - "Update InputMapping for for-body-graph failed, node:%s.", node->GetName().c_str()); + "[Update][InputMapping] for for-body-graph failed, node:%s.", node->GetName().c_str()); // for node has and only has one subgraph GE_CHECK_NOTNULL(node->GetOpDesc()); @@ -107,7 +109,9 @@ Status ForPass::BuildForInfo(const ComputeGraphPtr &root_graph, const NodePtr &n "in data anchor of op:%s(%s) lack, check invalid", FOR_START_INPUT, FOR_LIMIT_INPUT, FOR_DELTA_INPUT, node->GetName().c_str(), node->GetType().c_str()); - GELOGE(FAILED, "BuildForInfo for %s failed: start/limit/delta is NULL.", node->GetName().c_str()); + GELOGE(FAILED, "[Check][Param] FOR_START_INPUT index:%d or FOR_LIMIT_INPUT index:%d or FOR_DELTA_INPUT index:%d " + "in data anchor of op:%s(%s) lack", + FOR_START_INPUT, FOR_LIMIT_INPUT, FOR_DELTA_INPUT, node->GetName().c_str(), node->GetType().c_str()); return FAILED; } @@ -116,7 +120,7 @@ Status ForPass::BuildForInfo(const ComputeGraphPtr &root_graph, const NodePtr &n std::vector ctrl_inputs; std::vector ctrl_outputs; if (FindInputsAndOutputs(node, data_inputs, data_outputs, ctrl_inputs, ctrl_outputs) != SUCCESS) { - GELOGE(FAILED, "BuildForInfo for %s failed: find inputs/outputs failed.", node->GetName().c_str()); + GELOGE(FAILED, "[Find][InputsAndOutputs] in node:%s failed.", node->GetName().c_str()); return FAILED; } NodeUtils::UnlinkAll(*node); @@ -128,14 +132,16 @@ Status ForPass::BuildForInfo(const ComputeGraphPtr &root_graph, const NodePtr &n if (for_body_name.empty()) { REPORT_INNER_ERROR("E19999", "Get subgraph name from op:%s(%s) by index 0 failed", op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(FAILED, "BuildForInfo for %s failed: sub_graph_name is empty.", node->GetName().c_str()); + GELOGE(FAILED, "[Get][SubGraphName] from op:%s(%s) by index 0 failed", + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return FAILED; } ComputeGraphPtr for_body = root_graph->GetSubgraph(for_body_name); if (for_body == nullptr) { REPORT_INNER_ERROR("E19999", "Get subgraph from graph:%s by name:%s failed", root_graph->GetName().c_str(), for_body_name.c_str()); - GELOGE(FAILED, "BuildForInfo for %s failed: for_body_graph is NULL.", node->GetName().c_str()); + GELOGE(FAILED, "[Get][SubGraph] from graph:%s by name:%s failed", + root_graph->GetName().c_str(), for_body_name.c_str()); return FAILED; } @@ -162,13 +168,14 @@ Status ForPass::BuildForInfo(const ComputeGraphPtr &root_graph, const NodePtr &n /// OutDataAnchorPtr ForPass::FindInputWithIndex(const NodePtr &node, uint32_t index) { if (node == nullptr) { - GELOGE(FAILED, "FindInputWithIndex failed: node is NULL."); + GELOGE(FAILED, "[Check][Param] node is nullptr."); return nullptr; } InDataAnchorPtr in_data_anchor = node->GetInDataAnchor(index); if (in_data_anchor == nullptr) { - GELOGE(FAILED, "FindInputWithIndex %s:%u failed: in_data_anchor is NULL.", node->GetName().c_str(), index); + GELOGE(FAILED, "[Get][InDataAnchor] failed, In Data Anchor index:%u in node:%s is nullptr.", + index, node->GetName().c_str()); return nullptr; } @@ -234,9 +241,8 @@ Status ForPass::TranWhileInfo(const ComputeGraphPtr &graph, const ForInfo &for_i std::string i_name = for_name + "_i"; NodePtr i_node = graph->AddNode(CreateConstDesc(i_name, 0)); if (i_node == nullptr) { - REPORT_CALL_ERROR("E19999", "Add node:%s(Const) to graph:%s failed", - i_name.c_str(), graph->GetName().c_str()); - GELOGE(FAILED, "TranWhileInfo failed: create i_node failed."); + REPORT_CALL_ERROR("E19999", "Add node:%s(Const) to graph:%s failed", i_name.c_str(), graph->GetName().c_str()); + GELOGE(FAILED, "[Add][Node] %s(Const) to graph:%s failed", i_name.c_str(), graph->GetName().c_str()); return FAILED; } AddRePassNode(i_node); @@ -249,7 +255,9 @@ Status ForPass::TranWhileInfo(const ComputeGraphPtr &graph, const ForInfo &for_i REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(index:0) and op:%s(%s)(index:0) failed", i_node->GetName().c_str(), i_node->GetType().c_str(), identity_node->GetName().c_str(), identity_node->GetType().c_str()); - GELOGE(FAILED, "TranWhileInfo failed: Add data-edge %s:0->%s:0 failed.", i_name.c_str(), identity_name.c_str()); + GELOGE(FAILED, "[Add][Edge] between op:%s(%s)(index:0) and op:%s(%s)(index:0) failed", + i_node->GetName().c_str(), i_node->GetType().c_str(), + identity_node->GetName().c_str(), identity_node->GetType().c_str()); return FAILED; } AddRePassNode(identity_node); @@ -259,21 +267,22 @@ Status ForPass::TranWhileInfo(const ComputeGraphPtr &graph, const ForInfo &for_i if (i_input == nullptr) { REPORT_INNER_ERROR("E19999", "Out data anchor index:0 in op:%s(%s) is nullptr, check invalid", identity_node->GetName().c_str(), identity_node->GetType().c_str()); - GELOGE(FAILED, "TranWhileInfo failed: i_input is NULL."); + GELOGE(FAILED, "[Get][OutDataAnchor] failed, Out data anchor index:0 in op:%s(%s) is nullptr", + identity_node->GetName().c_str(), identity_node->GetType().c_str()); return FAILED; } OutDataAnchorPtr range_input = nullptr; OutDataAnchorPtr abs_delta_input = nullptr; if (CreateLoopInput(graph, for_info, range_input, abs_delta_input) != SUCCESS) { - GELOGE(FAILED, "TranWhileInfo failed: create loop input failed."); + GELOGE(FAILED, "[Create][LoopInput] failed, graph:%s.", graph->GetName().c_str()); return FAILED; } BuildWhileInfo(for_info, i_input, range_input, abs_delta_input, while_info); if (InsertWhileNode(graph, for_name + "_While", while_info) != SUCCESS) { - GELOGE(FAILED, "TranWhileInfo failed: insert while node failed."); + GELOGE(FAILED, "[Insert][WhileNode] in graph:%s failed.", graph->GetName().c_str()); return FAILED; } @@ -292,7 +301,7 @@ OpDescPtr ForPass::CreateConstDesc(const std::string &name, int32_t value) { OpDescPtr const_op_desc = MakeShared(name, CONSTANT); if (const_op_desc == nullptr) { REPORT_CALL_ERROR("E19999", "New OpDesc failed"); - GELOGE(FAILED, "Create op_desc failed, const:%s.", name.c_str()); + GELOGE(FAILED, "[New][OpDesc] failed."); return nullptr; } @@ -300,21 +309,23 @@ OpDescPtr ForPass::CreateConstDesc(const std::string &name, int32_t value) { GeTensorPtr const_value = MakeShared(data_desc, reinterpret_cast(&value), sizeof(int32_t)); if (const_value == nullptr) { REPORT_CALL_ERROR("E19999", "New GeTensor failed"); - GELOGE(FAILED, "Create tensor failed, const:%s.", name.c_str()); + GELOGE(FAILED, "[New][GeTensor] failed"); return nullptr; } if (!AttrUtils::SetTensor(const_op_desc, ATTR_NAME_WEIGHTS, const_value)) { REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed", ATTR_NAME_WEIGHTS.c_str(), const_op_desc->GetName().c_str(), const_op_desc->GetType().c_str()); - GELOGE(FAILED, "Set ATTR_NAME_WEIGHTS failed, const:%s.", name.c_str()); + GELOGE(FAILED, "[Set][Attr] %s to op:%s(%s) failed", ATTR_NAME_WEIGHTS.c_str(), + const_op_desc->GetName().c_str(), const_op_desc->GetType().c_str()); return nullptr; } if (const_op_desc->AddOutputDesc("y", data_desc) != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Add ouput desc to op:%s(%s) failed, name:y", const_op_desc->GetName().c_str(), const_op_desc->GetType().c_str()); - GELOGE(FAILED, "Add output desc failed, const:%s.", name.c_str()); + GELOGE(FAILED, "[Add][OutputDesc] to op:%s(%s) failed, name:y", + const_op_desc->GetName().c_str(), const_op_desc->GetType().c_str()); return nullptr; } @@ -360,7 +371,7 @@ Status ForPass::CreateLoopInput(const ComputeGraphPtr &graph, const ForInfo &for std::string error_msg; if ((graph_builder.Build(error_code, error_msg) == nullptr) || (error_code != GRAPH_SUCCESS)) { REPORT_CALL_ERROR("E19999", "Add loop input node to graph:%s failed", graph->GetName().c_str()); - GELOGE(FAILED, "Create loop_count node failed: error_code:%u, error_msg:%s.", error_code, error_msg.c_str()); + GELOGE(FAILED, "[Create][LoopInputNode] failed: error_code:%u, error_msg:%s.", error_code, error_msg.c_str()); return FAILED; } @@ -373,7 +384,7 @@ Status ForPass::CreateLoopInput(const ComputeGraphPtr &graph, const ForInfo &for NodePtr loop_count_node = graph_builder.GetNode(abs_name_1); if ((abs_delta_node == nullptr) || (loop_count_node == nullptr)) { REPORT_CALL_ERROR("E19999", "Add loop input node to graph:%s failed", graph->GetName().c_str()); - GELOGE(FAILED, "Create loop node failed: node is NULL."); + GELOGE(FAILED, "[Create][LoopNode] failed: node is nullptr, graph:%s.", graph->GetName().c_str()); return FAILED; } @@ -467,14 +478,15 @@ Status ForPass::InsertWhileNode(const ComputeGraphPtr &graph, const std::string if (while_node == nullptr) { REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s failed", op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str()); - GELOGE(FAILED, "Create while node failed, name:%s.", name.c_str()); + GELOGE(FAILED, "[Add][Node] %s(%s) to graph:%s failed", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str()); return FAILED; } AddRePassNode(while_node); while_info.while_node = while_node; if (BuildWhileLink(while_info) != SUCCESS) { - GELOGE(FAILED, "Build while link-edge failed, name:%s.", name.c_str()); + GELOGE(FAILED, "[Build][WhileLink] failed, node:%s.", while_node->GetName().c_str()); return FAILED; } @@ -500,7 +512,7 @@ Status ForPass::BuildWhileLink(const WhileInfo &while_info) { continue; } GE_CHK_GRAPH_STATUS_RET(GraphUtils::AddEdge(peer_out_anchor, in_data_anchor), - "Add data-edge %s:%d->%s:%zu failed.", + "[Add][DataEdge] %s:%d->%s:%zu failed.", peer_out_anchor->GetOwnerNode()->GetName().c_str(), peer_out_anchor->GetIdx(), while_node->GetName().c_str(), i); } @@ -511,7 +523,7 @@ Status ForPass::BuildWhileLink(const WhileInfo &while_info) { GE_CHECK_NOTNULL(out_data_anchor); for (auto &peer_in_anchor : while_info.data_outputs[i]) { GE_CHK_GRAPH_STATUS_RET(GraphUtils::AddEdge(out_data_anchor, peer_in_anchor), - "Add data-edge %s:%zu->%s:%d failed.", + "[Add][DataEdge] %s:%zu->%s:%d failed.", while_node->GetName().c_str(), i + kWhileOutputIndex, peer_in_anchor->GetOwnerNode()->GetName().c_str(), peer_in_anchor->GetIdx()); } @@ -521,7 +533,7 @@ Status ForPass::BuildWhileLink(const WhileInfo &while_info) { GE_CHECK_NOTNULL(in_ctrl_anchor); for (auto &peer_out_anchor : while_info.ctrl_inputs) { GE_CHK_GRAPH_STATUS_RET(GraphUtils::AddEdge(peer_out_anchor, in_ctrl_anchor), - "Add ctrl-edge %s->%s failed.", + "[Add][CtrlEdge] %s->%s failed.", peer_out_anchor->GetOwnerNode()->GetName().c_str(), in_ctrl_anchor->GetOwnerNode()->GetName().c_str()); } @@ -530,7 +542,7 @@ Status ForPass::BuildWhileLink(const WhileInfo &while_info) { GE_CHECK_NOTNULL(out_ctrl_anchor); for (auto &peer_in_anchor : while_info.ctrl_outputs) { GE_CHK_GRAPH_STATUS_RET(GraphUtils::AddEdge(out_ctrl_anchor, peer_in_anchor), - "Add ctrl-edge %s->%s failed.", + "[Add][CtrlEdge] %s->%s failed.", out_ctrl_anchor->GetOwnerNode()->GetName().c_str(), peer_in_anchor->GetOwnerNode()->GetName().c_str()); } @@ -585,7 +597,7 @@ ComputeGraphPtr ForPass::BuildCondGraph(WhileInfo &while_info) { ComputeGraphPtr cond_graph = graph_builder.Build(error_code, error_msg); if (cond_graph == nullptr) { REPORT_CALL_ERROR("E19999", "Build graph:%s failed", cond_name.c_str()); - GELOGE(FAILED, "Build cond_graph failed: error_code:%u, error_msg:%s.", error_code, error_msg.c_str()); + GELOGE(FAILED, "[Build][CondGraph] failed: error_code:%u, error_msg:%s.", error_code, error_msg.c_str()); return nullptr; } @@ -667,13 +679,13 @@ ComputeGraphPtr ForPass::BuildBodyGraph(WhileInfo &while_info) { std::string error_msg; ComputeGraphPtr body_graph = graph_builder.Build(error_code, error_msg); if (body_graph == nullptr) { - GELOGE(FAILED, "Build body_graph failed: error_code:%u, error_msg:%s.", error_code, error_msg.c_str()); + GELOGE(FAILED, "[Build][BodyGraph] failed: error_code:%u, error_msg:%s.", error_code, error_msg.c_str()); return nullptr; } NodePtr sub_graph_node = graph_builder.GetNode(sub_graph_node_name); if (sub_graph_node == nullptr) { - GELOGE(FAILED, "Get sub_graph_node failed: name:%s.", sub_graph_node_name.c_str()); + GELOGE(FAILED, "[Get][Node] by name:%s failed.", sub_graph_node_name.c_str()); return nullptr; } while_info.sub_graph_node = sub_graph_node; @@ -699,9 +711,8 @@ OpDescPtr ForPass::CreateSubgraphOpDesc(const std::string &name, uint32_t input_ OpDescPtr op_desc = op_desc_builder.Build(); if (op_desc == nullptr) { - REPORT_CALL_ERROR("E19999", "Build op_desc:%s(%s) failed", - name.c_str(), PARTITIONEDCALL); - GELOGE(FAILED, "Create op_desc for subgraph node failed, name:%s.", name.c_str()); + REPORT_CALL_ERROR("E19999", "Build op_desc:%s(%s) failed", name.c_str(), PARTITIONEDCALL); + GELOGE(FAILED, "[Build][OpDesc] %s(%s) failed", name.c_str(), PARTITIONEDCALL); return nullptr; } diff --git a/ge/graph/passes/fuse_data_nodes_with_common_input_pass.cc b/ge/graph/passes/fuse_data_nodes_with_common_input_pass.cc index 1f062813..ec7b2388 100644 --- a/ge/graph/passes/fuse_data_nodes_with_common_input_pass.cc +++ b/ge/graph/passes/fuse_data_nodes_with_common_input_pass.cc @@ -35,14 +35,14 @@ namespace ge { Status FuseDataNodesWithCommonInputPass::Run(ge::ComputeGraphPtr graph) { if (graph == nullptr) { REPORT_INNER_ERROR("E19999", "Param graph is nullptr, check invalid"); - GELOGE(GE_GRAPH_PARAM_NULLPTR, "Compute graph is null."); + GELOGE(GE_GRAPH_PARAM_NULLPTR, "[Check][Param] Compute graph is nullptr."); return GE_GRAPH_PARAM_NULLPTR; } GELOGD("FuseDataNodesWithCommonInputPass in."); // key: subgraph, value:--key: peer out anchor to parent node, --value: parent indexes to parent node map>> subgraphs_to_need_fuse_nodes_info; if (InitNeedFuseNodesInfo(graph, subgraphs_to_need_fuse_nodes_info) != SUCCESS) { - GELOGE(FAILED, "InitNeedFuseNodesInfo failed."); + GELOGE(FAILED, "[Init][NeedFuseNodesInfo] for graph:%s failed.", graph->GetName().c_str()); return FAILED; } return FuseDataNodes(subgraphs_to_need_fuse_nodes_info); @@ -116,7 +116,8 @@ Status FuseDataNodesWithCommonInputPass::FuseDataNodes( if (GraphUtils::RemoveNodeWithoutRelink(subgraph, node) != SUCCESS) { REPORT_CALL_ERROR("E19999", "Remove node:%s(%s) without relink in graph:%s failed", node->GetName().c_str(), node->GetType().c_str(), subgraph->GetName().c_str()); - GELOGE(FAILED, "[%s] RemoveNodeWithoutRelink failed.", node->GetName().c_str()); + GELOGE(FAILED, "[Remove][Node] %s(%s) without relink in graph:%s failed", + node->GetName().c_str(), node->GetType().c_str(), subgraph->GetName().c_str()); return FAILED; } } diff --git a/ge/graph/passes/get_original_format_pass.cc b/ge/graph/passes/get_original_format_pass.cc index 4b78ae49..670cd50c 100644 --- a/ge/graph/passes/get_original_format_pass.cc +++ b/ge/graph/passes/get_original_format_pass.cc @@ -37,7 +37,8 @@ using domi::SUCCESS; namespace ge { Status GetOriginalFormatPass::Run(ge::ComputeGraphPtr graph) { GE_CHECK_NOTNULL(graph); - GE_RETURN_WITH_LOG_IF_ERROR(SetOriginalFormat(graph), "SetOriginalFormat failed"); + GE_RETURN_WITH_LOG_IF_ERROR(SetOriginalFormat(graph), + "[Set][OriginalFormat] for graph:%s failed", graph->GetName().c_str()); return SUCCESS; } @@ -54,7 +55,8 @@ Status GetOriginalFormatPass::SetOriginalFormat(const ge::ComputeGraphPtr &graph REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed", ATTR_NAME_INFERRED_FORMAT.c_str(), node_ptr->GetName().c_str(), node_ptr->GetType().c_str()); - GELOGE(FAILED, "set ATTR_NAME_INFERRED_FORMAT failed"); + GELOGE(FAILED, "[Set][Attr] %s to op:%s(%s) failed", ATTR_NAME_INFERRED_FORMAT.c_str(), + node_ptr->GetName().c_str(), node_ptr->GetType().c_str()); return FAILED); } @@ -70,13 +72,15 @@ Status GetOriginalFormatPass::SetOriginalFormat(const ge::ComputeGraphPtr &graph REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed", ATTR_NAME_FORMAT.c_str(), desc_ptr->GetName().c_str(), desc_ptr->GetType().c_str()); - GELOGE(FAILED, "set ATTR_NAME_FORMAT failed"); + GELOGE(FAILED, "[Set][Attr] %s to op:%s(%s) failed", ATTR_NAME_FORMAT.c_str(), + desc_ptr->GetName().c_str(), desc_ptr->GetType().c_str()); return FAILED); GE_IF_BOOL_EXEC(!AttrUtils::SetInt(desc_ptr, ATTR_NAME_INFERRED_FORMAT, ori_format), REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed", ATTR_NAME_INFERRED_FORMAT.c_str(), desc_ptr->GetName().c_str(), desc_ptr->GetType().c_str()); - GELOGE(FAILED, "set ATTR_NAME_INFERRED_FORMAT failed"); + GELOGE(FAILED, "[Set][Attr] %s to op:%s(%s) failed", ATTR_NAME_INFERRED_FORMAT.c_str(), + desc_ptr->GetName().c_str(), desc_ptr->GetType().c_str()); return FAILED); continue; } @@ -142,7 +146,8 @@ Status GetOriginalFormatPass::SetOriginalFormat(const ge::ComputeGraphPtr &graph REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed", ATTR_NAME_IGNORE_PRED_FORMAT.c_str(), tmp_op_ptr->GetName().c_str(), tmp_op_ptr->GetType().c_str()); - GELOGE(FAILED, "remove edge failed"); + GELOGE(FAILED, "[Set][Attr] %s to op:%s(%s) failed", ATTR_NAME_IGNORE_PRED_FORMAT.c_str(), + tmp_op_ptr->GetName().c_str(), tmp_op_ptr->GetType().c_str()); return FAILED); } @@ -152,23 +157,27 @@ Status GetOriginalFormatPass::SetOriginalFormat(const ge::ComputeGraphPtr &graph REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed", ATTR_NAME_FORMAT.c_str(), tmp_op_ptr->GetName().c_str(), tmp_op_ptr->GetType().c_str()); - GELOGE(FAILED, "set ATTR_NAME_FORMAT failed"); + GELOGE(FAILED, "[Set][Attr] %s to op:%s(%s) failed", ATTR_NAME_FORMAT.c_str(), + tmp_op_ptr->GetName().c_str(), tmp_op_ptr->GetType().c_str()); return FAILED); GE_IF_BOOL_EXEC(!AttrUtils::SetInt(tmp_op_ptr, ATTR_NAME_INFERRED_FORMAT, ori_format), REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed", ATTR_NAME_INFERRED_FORMAT.c_str(), tmp_op_ptr->GetName().c_str(), tmp_op_ptr->GetType().c_str()); - GELOGE(FAILED, "set ATTR_NAME_INFERRED_FORMAT failed"); + GELOGE(FAILED, "[Set][Attr] %s to op:%s(%s) failed", ATTR_NAME_INFERRED_FORMAT.c_str(), + tmp_op_ptr->GetName().c_str(), tmp_op_ptr->GetType().c_str()); return FAILED); } else { int64_t existingFormat = 0; GE_RETURN_WITH_LOG_IF_FALSE(AttrUtils::GetInt(tmp_op_ptr, ATTR_NAME_FORMAT, existingFormat), - "Get existing_format attr failed"); + "[Get][Attr] %s from op:%s(%s) failed", ATTR_NAME_FORMAT.c_str(), + tmp_op_ptr->GetName().c_str(), tmp_op_ptr->GetType().c_str()); if (!AttrUtils::SetInt(tmp_op_ptr, ATTR_NAME_INFERRED_FORMAT, existingFormat)) { REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed", ATTR_NAME_INFERRED_FORMAT.c_str(), tmp_op_ptr->GetName().c_str(), tmp_op_ptr->GetType().c_str()); - GELOGE(FAILED, "set ATTR_NAME_INFERRED_FORMAT failed"); + GELOGE(FAILED, "[Set][Attr] %s to op:%s(%s) failed", ATTR_NAME_INFERRED_FORMAT.c_str(), + tmp_op_ptr->GetName().c_str(), tmp_op_ptr->GetType().c_str()); return FAILED; } } @@ -177,7 +186,7 @@ Status GetOriginalFormatPass::SetOriginalFormat(const ge::ComputeGraphPtr &graph } bool GetOriginalFormatPass::IsFormatTranspose(const ge::OpDescPtr op_ptr, int32_t ori_format) { - GE_CHK_BOOL_EXEC(op_ptr != nullptr, return false, "opdef is nullptr"); + GE_CHK_BOOL_EXEC(op_ptr != nullptr, return false, "[Check][Param] op_ptr is nullptr"); if (op_ptr->GetType() == PERMUTE) { vector index_list; GE_IF_BOOL_EXEC(!AttrUtils::GetListInt(op_ptr, PERMUTE_ATTR_ORDER, index_list), return false); diff --git a/ge/graph/passes/global_step_insert_pass.cc b/ge/graph/passes/global_step_insert_pass.cc index d702e758..f27641fc 100755 --- a/ge/graph/passes/global_step_insert_pass.cc +++ b/ge/graph/passes/global_step_insert_pass.cc @@ -41,7 +41,7 @@ NodePtr GlobalStepInsertPass::InsertOp(ComputeGraphPtr &compute_graph, OpDescPtr op_desc = MakeShared(node_name, node_type); GE_IF_BOOL_EXEC(op_desc == nullptr, REPORT_CALL_ERROR("E19999", "New OpDesc failed"); - GELOGE(FAILED,"Make OpDesc failed"); + GELOGE(FAILED,"[New][OpDesc] failed"); return nullptr); for (auto &input_desc : input_list) { @@ -49,7 +49,8 @@ NodePtr GlobalStepInsertPass::InsertOp(ComputeGraphPtr &compute_graph, if (graph_status != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Add input desc to op:%s(%s) failed", op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(FAILED, "Add node:%s intput desc failed, error=%u.", node_name.c_str(), graph_status); + GELOGE(FAILED, "[Add][InputDesc] to op:%s(%s) failed", + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return nullptr; } } @@ -59,17 +60,19 @@ NodePtr GlobalStepInsertPass::InsertOp(ComputeGraphPtr &compute_graph, if (graph_status != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Add output desc to op:%s(%s) failed", op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(FAILED, "Add node:%s output desc failed, error=%u.", node_name.c_str(), graph_status); + GELOGE(FAILED, "[Add][OutputDesc] to op:%s(%s) failed, ret:%u.", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph_status); return nullptr; } } - GE_IF_BOOL_EXEC(compute_graph == nullptr, GELOGE(FAILED,"compute_graph is nullptr"); return nullptr); + GE_IF_BOOL_EXEC(compute_graph == nullptr, GELOGE(FAILED,"[Check][Param] compute_graph is nullptr"); return nullptr); NodePtr node = compute_graph->AddNode(op_desc); GE_IF_BOOL_EXEC(node == nullptr, REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s failed", op_desc->GetName().c_str(), op_desc->GetType().c_str(), compute_graph->GetName().c_str()); - GELOGE(FAILED, "add node failed, name:%s, type:%s.", node_name.c_str(), node_type.c_str()); + GELOGE(FAILED, "[Add][Node] %s(%s) to graph:%s failed", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), compute_graph->GetName().c_str()); return nullptr); GELOGI("Insert op success, name:%s, type:%s.", node_name.c_str(), node_type.c_str()); @@ -107,7 +110,7 @@ Status GlobalStepInsertPass::Run(ComputeGraphPtr compute_graph) { NodePtr global_step = InsertOp(compute_graph, VARIABLE, NODE_NAME_GLOBAL_STEP, input_desc_list, output_desc_list); if (global_step == nullptr) { - GELOGE(FAILED, "Add global_step node failed, global_step is null."); + GELOGE(FAILED, "[Insert][Op] to graph:%s failed.", compute_graph->GetName().c_str()); return FAILED; } @@ -117,7 +120,9 @@ Status GlobalStepInsertPass::Run(ComputeGraphPtr compute_graph) { REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed", global_step->GetName().c_str(), global_step->GetType().c_str(), output_node->GetName().c_str(), output_node->GetType().c_str()); - GELOGE(FAILED, "Add global_step to netoutput edge failed, add_ret=%u.", add_ret); + GELOGE(FAILED, "[Add][ControlEdge] between op:%s(%s) and op:%s(%s) failed", + global_step->GetName().c_str(), global_step->GetType().c_str(), + output_node->GetName().c_str(), output_node->GetType().c_str()); return FAILED; } GELOGD("Add global_step to netoutput edge in graph %u success", compute_graph->GetGraphID()); diff --git a/ge/graph/passes/guarantee_const_pass.cc b/ge/graph/passes/guarantee_const_pass.cc index f6567fce..1d369f38 100644 --- a/ge/graph/passes/guarantee_const_pass.cc +++ b/ge/graph/passes/guarantee_const_pass.cc @@ -36,19 +36,20 @@ Status GuaranteeConstPass::Run(NodePtr &node) { string type; Status status_ret = GetOriginalType(node, type); if (status_ret != SUCCESS) { - REPORT_CALL_ERROR("E19999", "Get original type for node:%s failed", - node->GetName().c_str()); - GELOGE(status_ret, "GuaranteeConstPass get original type fail."); + REPORT_CALL_ERROR("E19999", "Get original type for node:%s failed", node->GetName().c_str()); + GELOGE(status_ret, "[Get][OriginalType] for node:%s failed", node->GetName().c_str()); return status_ret; } if (type != GUARANTEECONST) { return SUCCESS; } if (node->GetOpDesc()->GetAllInputsDesc().size() != kGuaranteeConstInputsSize) { - REPORT_CALL_ERROR("E19999", "Num:%zu of input desc node:%s(%s) not equal to %u, " + REPORT_CALL_ERROR("E19999", "Num:%zu of input desc in node:%s(%s) not equal to %u, " "check invalid", node->GetOpDesc()->GetAllInputsDesc().size(), node->GetName().c_str(), node->GetType().c_str(), kGuaranteeConstInputsSize); - GELOGE(PARAM_INVALID, "input size error. Input size:%zu", node->GetOpDesc()->GetAllInputsDesc().size()); + GELOGE(PARAM_INVALID, "[Check][Param] Num:%zu of input desc in node:%s(%s) not equal to %u", + node->GetOpDesc()->GetAllInputsDesc().size(), + node->GetName().c_str(), node->GetType().c_str(), kGuaranteeConstInputsSize); return PARAM_INVALID; } // [Cascade pointer] @@ -57,12 +58,13 @@ Status GuaranteeConstPass::Run(NodePtr &node) { // Input tensor cannot be a resource variable handle. const DataType &input_dtype = in_desc->GetDataType(); if (input_dtype == DT_RESOURCE) { - REPORT_CALL_ERROR("E19999", - "Data type:%s of op:%s(%s) input0 tensor not equal to %s, check invalid", + REPORT_CALL_ERROR("E19999", "Data type:%s of op:%s(%s) input0 tensor not equal to %s, check invalid", TypeUtils::DataTypeToSerialString(input_dtype).c_str(), node->GetName().c_str(), node->GetType().c_str(), TypeUtils::DataTypeToSerialString(DT_RESOURCE).c_str()); - GELOGE(FAILED, "Input tensor cannot be a resource variable handle in [%s].", node->GetName().c_str()); + GELOGE(FAILED, "[Check][Param] Data type:%s of op:%s(%s) input0 tensor not equal to %s", + TypeUtils::DataTypeToSerialString(input_dtype).c_str(), + node->GetName().c_str(), node->GetType().c_str(), TypeUtils::DataTypeToSerialString(DT_RESOURCE).c_str()); return FAILED; } diff --git a/ge/graph/passes/hccl_continuous_memcpy_pass.cc b/ge/graph/passes/hccl_continuous_memcpy_pass.cc index 790661bc..61066d63 100644 --- a/ge/graph/passes/hccl_continuous_memcpy_pass.cc +++ b/ge/graph/passes/hccl_continuous_memcpy_pass.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,11 +24,12 @@ #include "common/ge/ge_util.h" #include "framework/common/types.h" #include "graph/utils/graph_utils.h" +#include "graph/utils/op_desc_utils.h" namespace { -const int kAnchorNum = 0; const int32_t kAnchorAssignRefIndex = 0; const int32_t kAnchorAssignValueIndex = 1; +const int32_t kAnchorIdentityIndex = 0; } // namespace namespace ge { Status HcclContinuousMemcpyPass::Run(ge::ComputeGraphPtr graph) { @@ -161,41 +162,23 @@ NodePtr HcclContinuousMemcpyPass::CreateIdentityNode(const ComputeGraphPtr &grap std::string node_name = pre_node->GetName() + "_" + IDENTITY; node_name = CheckDuplicateName(node_name); - OpDescPtr op_desc = MakeShared(node_name.c_str(), IDENTITY); - if (op_desc == nullptr) { - REPORT_CALL_ERROR("E19999", "New OpDesc failed"); - GELOGE(INTERNAL_ERROR, "Create Identity op: MakeShared op_desc fail."); - return nullptr; - } - GELOGI("Create Identity op:%s.", op_desc->GetName().c_str()); - - graphStatus ret = op_desc->AddInputDesc("x", pre_op_desc->GetOutputDesc(out_data_anchor->GetIdx())); - if (ret != GRAPH_SUCCESS) { - REPORT_CALL_ERROR("E19999", "Add input desc to op:%s(%s) failed", - op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Create Identity op: add input desc fail."); - return nullptr; - } - - ret = op_desc->AddOutputDesc("y", pre_op_desc->GetOutputDesc(out_data_anchor->GetIdx())); - if (ret != GRAPH_SUCCESS) { - REPORT_CALL_ERROR("E19999", "Add output desc to op:%s(%s) failed", - op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Create Identity op: add output desc fail."); + OpDescBuilder op_desc_builder(node_name, IDENTITY); + auto data_desc = pre_op_desc->GetOutputDesc(out_data_anchor->GetIdx()); + auto identity_op_desc = op_desc_builder.AddInput("x", data_desc).AddOutput("y", data_desc).Build(); + if (identity_op_desc == nullptr) { return nullptr; } // because history reason ,this pass can not do work after constant fold so mark it - (void)AttrUtils::SetBool(op_desc, ATTR_NO_NEED_CONSTANT_FOLDING, false); + (void)AttrUtils::SetBool(identity_op_desc, ATTR_NO_NEED_CONSTANT_FOLDING, false); - NodePtr memcpy_node = graph->AddNode(op_desc); - if (memcpy_node == nullptr) { + NodePtr identity_node = graph->AddNode(identity_op_desc); + if (identity_node == nullptr) { REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s failed", - op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str()); + identity_node->GetName().c_str(), identity_node->GetType().c_str(), graph->GetName().c_str()); GELOGE(INTERNAL_ERROR, "Insert Identity node fail."); return nullptr; } - - return memcpy_node; + return identity_node; } /// @@ -256,50 +239,24 @@ Status HcclContinuousMemcpyPass::ModifyEdgeConnection(const ComputeGraphPtr &gra Status HcclContinuousMemcpyPass::InsertIdentityBeforeHccl(const ComputeGraphPtr &graph, const OutDataAnchorPtr &src_out_anchor, const InDataAnchorPtr &hccl_in_anchor) { - GELOGI("Between op %s and op %s need insert memcpy async op.", src_out_anchor->GetOwnerNode()->GetName().c_str(), + GELOGI("Between op %s and op %s need insert identity op.", src_out_anchor->GetOwnerNode()->GetName().c_str(), hccl_in_anchor->GetOwnerNode()->GetName().c_str()); - NodePtr memcpy_node = CreateIdentityNode(graph, src_out_anchor); - GE_CHECK_NOTNULL(memcpy_node); + NodePtr identity_node = CreateIdentityNode(graph, src_out_anchor); + GE_CHECK_NOTNULL(identity_node); - Status ret1 = src_out_anchor->Unlink(hccl_in_anchor); - if (ret1 != SUCCESS) { - REPORT_CALL_ERROR("E19999", - "Op:%s(%s) out index:%d unlink from op:%s(%s) in index:%d failed", - src_out_anchor->GetOwnerNode()->GetName().c_str(), - src_out_anchor->GetOwnerNode()->GetType().c_str(), src_out_anchor->GetIdx(), - hccl_in_anchor->GetOwnerNode()->GetName().c_str(), - hccl_in_anchor->GetOwnerNode()->GetType().c_str(), - hccl_in_anchor->GetIdx()); - GELOGE(INTERNAL_ERROR, "The op %s Unlink anchor %s fail.", src_out_anchor->GetOwnerNode()->GetName().c_str(), - hccl_in_anchor->GetOwnerNode()->GetName().c_str()); - return FAILED; - } - auto out_data_anchor_0 = memcpy_node->GetOutDataAnchor(kAnchorNum); - GE_CHECK_NOTNULL(out_data_anchor_0); - ret1 = out_data_anchor_0->LinkTo(hccl_in_anchor); - if (ret1 != SUCCESS) { + auto ret = GraphUtils::InsertNodeBefore(hccl_in_anchor, identity_node, kAnchorIdentityIndex, kAnchorIdentityIndex); + if (ret != SUCCESS) { REPORT_CALL_ERROR("E19999", - "Op:%s(%s) out index:%d link to op:%s(%s) in index:%d failed", - out_data_anchor_0->GetOwnerNode()->GetName().c_str(), - out_data_anchor_0->GetOwnerNode()->GetType().c_str(), out_data_anchor_0->GetIdx(), + "Op:Fail to insert %s(%s) before %s(%s) on index:%d input anchor.", + identity_node->GetName().c_str(), identity_node->GetType().c_str(), hccl_in_anchor->GetOwnerNode()->GetName().c_str(), hccl_in_anchor->GetOwnerNode()->GetType().c_str(), hccl_in_anchor->GetIdx()); - GELOGE(INTERNAL_ERROR, "The op %s link anchor %s fail.", memcpy_node->GetName().c_str(), - hccl_in_anchor->GetOwnerNode()->GetName().c_str()); - return FAILED; - } - - Status ret = src_out_anchor->LinkTo(memcpy_node->GetInDataAnchor(kAnchorNum)); - if (ret != SUCCESS) { - REPORT_CALL_ERROR("E19999", - "Op:%s(%s) out index:%d link to op:%s(%s) in index:%u failed", - src_out_anchor->GetOwnerNode()->GetName().c_str(), - src_out_anchor->GetOwnerNode()->GetType().c_str(), src_out_anchor->GetIdx(), - memcpy_node->GetName().c_str(), memcpy_node->GetType().c_str(), - kAnchorNum); - GELOGE(INTERNAL_ERROR, "The op %s link anchor %s fail.", src_out_anchor->GetOwnerNode()->GetName().c_str(), - memcpy_node->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "Fail to insert %s(%s) before %s(%s) on index:%d input anchor.", + identity_node->GetName().c_str(), identity_node->GetType().c_str(), + hccl_in_anchor->GetOwnerNode()->GetName().c_str(), + hccl_in_anchor->GetOwnerNode()->GetType().c_str(), + hccl_in_anchor->GetIdx()); return FAILED; } return SUCCESS; diff --git a/ge/graph/passes/hccl_continuous_memcpy_pass.h b/ge/graph/passes/hccl_continuous_memcpy_pass.h index 538e89e9..5fbb6fd0 100644 --- a/ge/graph/passes/hccl_continuous_memcpy_pass.h +++ b/ge/graph/passes/hccl_continuous_memcpy_pass.h @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/ge/graph/passes/hccl_memcpy_pass.cc b/ge/graph/passes/hccl_memcpy_pass.cc index 2d2f8220..125780fa 100755 --- a/ge/graph/passes/hccl_memcpy_pass.cc +++ b/ge/graph/passes/hccl_memcpy_pass.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,13 +24,15 @@ #include "common/ge/ge_util.h" #include "framework/common/types.h" #include "graph/utils/graph_utils.h" +#include "graph/utils/op_desc_utils.h" namespace { const int32_t kAnchorSize = 1; -const int kAnchorNum = 0; const int32_t kAnchorAssignRefIndex = 0; const int32_t kAnchorAssignValueIndex = 1; -const char *const kInputMutable = "_input_mutable"; +const int32_t kAnchorIdentityIndex = 0; +// attr _input_mutable = true means hccl node will modify its input in runtime +const char *const kModifyInput = "_input_mutable"; } // namespace namespace ge { Status HcclMemcpyPass::Run(ge::ComputeGraphPtr graph) { @@ -58,24 +60,13 @@ Status HcclMemcpyPass::Run(ge::ComputeGraphPtr graph) { // need to inset memcpy node between. // also works on situation that input is variable or const. Status HcclMemcpyPass::MutableInputProcess(const ComputeGraphPtr &graph, const NodePtr node) { - auto op_desc = node->GetOpDesc(); - bool node_input_mutable = false; - if (!AttrUtils::HasAttr(op_desc, kInputMutable)) { - return SUCCESS; - } - - if (!AttrUtils::GetBool(op_desc, kInputMutable, node_input_mutable)) { - REPORT_CALL_ERROR("E19999", "Get Attr:%s from op:%s(%s) failed", kInputMutable, - op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "node:%s get attr:_input_mutable failed.", node->GetName().c_str()); - return FAILED; - } + (void)AttrUtils::GetBool(node->GetOpDesc(), kModifyInput, node_input_mutable); if (!node_input_mutable) { return SUCCESS; } - GELOGI("input mutable hcom op is:%s.", op_desc->GetName().c_str()); + GELOGI("input mutable hcom op is:%s.", node->GetName().c_str()); for (auto &hccl_in_anchor : node->GetAllInDataAnchors()) { if (hccl_in_anchor == nullptr) { continue; @@ -127,41 +118,23 @@ NodePtr HcclMemcpyPass::CreateIdentityNode(const ComputeGraphPtr &graph, const O std::string node_name = pre_node->GetName() + "_" + IDENTITY; node_name = CheckDuplicateName(node_name); - OpDescPtr op_desc = MakeShared(node_name.c_str(), IDENTITY); - if (op_desc == nullptr) { - REPORT_CALL_ERROR("E19999", "New OpDesc failed"); - GELOGE(INTERNAL_ERROR, "Create Identity op: MakeShared op_desc fail."); - return nullptr; - } - GELOGI("Create Identity op:%s.", op_desc->GetName().c_str()); - - graphStatus ret = op_desc->AddInputDesc("x", pre_op_desc->GetOutputDesc(out_data_anchor->GetIdx())); - if (ret != GRAPH_SUCCESS) { - REPORT_CALL_ERROR("E19999", "Add input desc to op:%s(%s) failed, name:x", - op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Create Identity op: add input desc fail."); - return nullptr; - } - - ret = op_desc->AddOutputDesc("y", pre_op_desc->GetOutputDesc(out_data_anchor->GetIdx())); - if (ret != GRAPH_SUCCESS) { - REPORT_CALL_ERROR("E19999", "Add output desc to op:%s(%s) failed, name:y", - op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Create Identity op: add output desc fail."); + OpDescBuilder op_desc_builder(node_name, IDENTITY); + auto data_desc = pre_op_desc->GetOutputDesc(out_data_anchor->GetIdx()); + auto identity_op_desc = op_desc_builder.AddInput("x", data_desc).AddOutput("y", data_desc).Build(); + if (identity_op_desc == nullptr) { return nullptr; } // because history reason ,this pass can not do work after constant fold so mark it - (void)AttrUtils::SetBool(op_desc, ATTR_NO_NEED_CONSTANT_FOLDING, false); + (void)AttrUtils::SetBool(identity_op_desc, ATTR_NO_NEED_CONSTANT_FOLDING, false); - NodePtr memcpy_node = graph->AddNode(op_desc); - if (memcpy_node == nullptr) { + NodePtr identity_node = graph->AddNode(identity_op_desc); + if (identity_node == nullptr) { REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s failed", - op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str()); + identity_node->GetName().c_str(), identity_node->GetType().c_str(), graph->GetName().c_str()); GELOGE(INTERNAL_ERROR, "Insert Identity node fail."); return nullptr; } - - return memcpy_node; + return identity_node; } /// @@ -220,49 +193,24 @@ Status HcclMemcpyPass::ModifyEdgeConnection(const ComputeGraphPtr &graph, const /// Status HcclMemcpyPass::InsertIdentityBeforeHccl(const ComputeGraphPtr &graph, const OutDataAnchorPtr &src_out_anchor, const InDataAnchorPtr &hccl_in_anchor) { - GELOGI("Between op %s and op %s need insert memcpy async op.", src_out_anchor->GetOwnerNode()->GetName().c_str(), + GELOGI("Between op %s and op %s need insert identity op.", src_out_anchor->GetOwnerNode()->GetName().c_str(), hccl_in_anchor->GetOwnerNode()->GetName().c_str()); - NodePtr memcpy_node = CreateIdentityNode(graph, src_out_anchor); - GE_CHECK_NOTNULL(memcpy_node); + NodePtr identity_node = CreateIdentityNode(graph, src_out_anchor); + GE_CHECK_NOTNULL(identity_node); - Status ret1 = src_out_anchor->Unlink(hccl_in_anchor); - if (ret1 != SUCCESS) { - REPORT_CALL_ERROR("E19999", - "Op:%s(%s) out index:%d unlink from op:%s(%s) in index:%d failed", - src_out_anchor->GetOwnerNode()->GetName().c_str(), - src_out_anchor->GetOwnerNode()->GetType().c_str(), src_out_anchor->GetIdx(), - hccl_in_anchor->GetOwnerNode()->GetName().c_str(), - hccl_in_anchor->GetOwnerNode()->GetType().c_str(), hccl_in_anchor->GetIdx()); - GELOGE(INTERNAL_ERROR, "The op %s Unlink anchor %s fail.", src_out_anchor->GetOwnerNode()->GetName().c_str(), - hccl_in_anchor->GetOwnerNode()->GetName().c_str()); - return FAILED; - } - auto out_data_anchor_0 = memcpy_node->GetOutDataAnchor(kAnchorNum); - GE_CHECK_NOTNULL(out_data_anchor_0); - ret1 = out_data_anchor_0->LinkTo(hccl_in_anchor); - if (ret1 != SUCCESS) { + auto ret = GraphUtils::InsertNodeBefore(hccl_in_anchor, identity_node, kAnchorIdentityIndex, kAnchorIdentityIndex); + if (ret != SUCCESS) { REPORT_CALL_ERROR("E19999", - "Op:%s(%s) out index:%d link to op:%s(%s) in index:%d failed", - out_data_anchor_0->GetOwnerNode()->GetName().c_str(), - out_data_anchor_0->GetOwnerNode()->GetType().c_str(), out_data_anchor_0->GetIdx(), + "Op:Fail to insert %s(%s) before %s(%s) on index:%d input anchor.", + identity_node->GetName().c_str(), identity_node->GetType().c_str(), hccl_in_anchor->GetOwnerNode()->GetName().c_str(), hccl_in_anchor->GetOwnerNode()->GetType().c_str(), hccl_in_anchor->GetIdx()); - GELOGE(INTERNAL_ERROR, "The op %s link anchor %s fail.", memcpy_node->GetName().c_str(), - hccl_in_anchor->GetOwnerNode()->GetName().c_str()); - return FAILED; - } - - Status ret = src_out_anchor->LinkTo(memcpy_node->GetInDataAnchor(kAnchorNum)); - if (ret != SUCCESS) { - REPORT_CALL_ERROR("E19999", - "Op:%s(%s) out index:%d link to op:%s(%s) in index:%u failed", - src_out_anchor->GetOwnerNode()->GetName().c_str(), - src_out_anchor->GetOwnerNode()->GetType().c_str(), src_out_anchor->GetIdx(), - memcpy_node->GetName().c_str(), memcpy_node->GetType().c_str(), - kAnchorNum); - GELOGE(INTERNAL_ERROR, "The op %s link anchor %s fail.", src_out_anchor->GetOwnerNode()->GetName().c_str(), - memcpy_node->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "Fail to insert %s(%s) before %s(%s) on index:%d input anchor.", + identity_node->GetName().c_str(), identity_node->GetType().c_str(), + hccl_in_anchor->GetOwnerNode()->GetName().c_str(), + hccl_in_anchor->GetOwnerNode()->GetType().c_str(), + hccl_in_anchor->GetIdx()); return FAILED; } return SUCCESS; @@ -340,13 +288,13 @@ Status HcclMemcpyPass::InsertAssignAfterBroadcastIfNeed(const ComputeGraphPtr &g } ret = assign_out_control_anchor->LinkTo(in_data_anchor->GetOwnerNode()->GetInControlAnchor()); if (ret != SUCCESS) { - REPORT_CALL_ERROR("E19999", - "Op:%s(%s) out index:%d link to op:%s(%s) in index:%d failed", - assign_out_control_anchor->GetOwnerNode()->GetName().c_str(), - assign_out_control_anchor->GetOwnerNode()->GetType().c_str(), assign_out_control_anchor->GetIdx(), - in_data_anchor->GetOwnerNode()->GetName().c_str(), - in_data_anchor->GetOwnerNode()->GetType().c_str(), - in_data_anchor->GetIdx()); + REPORT_CALL_ERROR("E19999", "Op:%s(%s) out index:%d link to op:%s(%s) in index:%d failed", + assign_out_control_anchor->GetOwnerNode()->GetName().c_str(), + assign_out_control_anchor->GetOwnerNode()->GetType().c_str(), + assign_out_control_anchor->GetIdx(), + in_data_anchor->GetOwnerNode()->GetName().c_str(), + in_data_anchor->GetOwnerNode()->GetType().c_str(), + in_data_anchor->GetIdx()); GELOGE(INTERNAL_ERROR, "The op %s link control anchor %s fail.", assign_out_control_anchor->GetOwnerNode()->GetName().c_str(), in_data_anchor->GetOwnerNode()->GetName().c_str()); diff --git a/ge/graph/passes/hccl_memcpy_pass.h b/ge/graph/passes/hccl_memcpy_pass.h index 7ab63c59..b75b27d1 100755 --- a/ge/graph/passes/hccl_memcpy_pass.h +++ b/ge/graph/passes/hccl_memcpy_pass.h @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/ge/graph/passes/inplace_support_check_pass.cc b/ge/graph/passes/inplace_support_check_pass.cc index 9f0d76d0..abdb4001 100644 --- a/ge/graph/passes/inplace_support_check_pass.cc +++ b/ge/graph/passes/inplace_support_check_pass.cc @@ -78,8 +78,6 @@ Status InplaceSupportCheckPass::Run(NodePtr &node) { AddRePassNode(node); break; } - - GELOGD("InplaceSupportCheckPass success"); return SUCCESS; } } // namespace ge diff --git a/ge/graph/passes/mark_force_unknown_for_cond_pass.cc b/ge/graph/passes/mark_force_unknown_for_cond_pass.cc index 6729a647..f6c87d58 100644 --- a/ge/graph/passes/mark_force_unknown_for_cond_pass.cc +++ b/ge/graph/passes/mark_force_unknown_for_cond_pass.cc @@ -18,20 +18,25 @@ #include +#include "graph/utils/node_utils.h" #include "graph/common/omg_util.h" namespace ge { namespace { -const std::set kMergeOpTypes{ MERGE, REFMERGE }; +inline bool IsMergeInLoop(const NodePtr &node) { + const static std::set kLoopMergeInputs{ ENTER, REFENTER, NEXTITERATION, REFNEXTITERATION }; -const std::set kSwitchOpTypes{ SWITCH, REFSWITCH }; + std::string node_type; + (void)GetOriginalType(node, node_type); + return kLoopMergeInputs.count(node_type) > 0; +} -const std::set kLoopMergeInputs{ ENTER, REFENTER, NEXTITERATION, REFNEXTITERATION }; +inline bool IsSwitchInLoop(const NodePtr &node) { + const static std::set kLoopSwitchInputs{ MERGE, REFMERGE, LOOPCOND }; -inline bool IsMergeInLoop(const NodePtr &node) { std::string node_type; (void)GetOriginalType(node, node_type); - return kLoopMergeInputs.count(node_type) > 0; + return kLoopSwitchInputs.count(node_type) > 0; } } @@ -103,7 +108,13 @@ void MarkForceUnknownForCondPass::MarkUnknownForSwitch(const NodePtr &node, std: if (dst_span > 0) { search_queue.push({in_node, dst_span - 1}); } else { - switch_group.emplace_back(in_node); + const auto &all_in_nodes = in_node->GetInDataNodes(); + if (std::any_of(all_in_nodes.begin(), all_in_nodes.end(), IsSwitchInLoop)) { + GELOGW("Travel node: %s, %s node: %s, Skip LoopCond switch", dst_node->GetName().c_str(), node_type.c_str(), + in_node->GetName().c_str()); + } else { + switch_group.emplace_back(in_node); + } } } else if (kMergeOpTypes.count(node_type) > 0) { // Merge input node. search_queue.push({in_node, dst_span + 1}); @@ -121,19 +132,37 @@ void MarkForceUnknownForCondPass::MarkUnknownForSwitch(const NodePtr &node, std: /// void MarkForceUnknownForCondPass::MarkUnknownForSwitch(const std::map> &switch_groups) { std::function callback = [](const NodePtr &n) { - return n->GetOpDesc()->HasAttr(ATTR_NAME_FORCE_UNKNOWN_SHAPE); + return n->GetOpDesc()->HasAttr(ATTR_NAME_CONTROL_FLOW_GROUP); }; - for (const auto &group : switch_groups) { - const auto &node = group.first; - const auto &switch_group = group.second; - const auto &op_desc = node->GetOpDesc(); - if (IsUnknownShapeTensor(op_desc->GetOutputDesc(0)) || op_desc->HasAttr(ATTR_NAME_FORCE_UNKNOWN_SHAPE) || - std::any_of(switch_group.begin(), switch_group.end(), callback)) { - GELOGI("Mark [%s] as force unknown shape", node->GetName().c_str()); - MarkForceUnknownShape(node, true); - for (const auto &n : switch_group) { - MarkForceUnknownShape(n, true); + for (auto it1 = switch_groups.begin(); it1 != switch_groups.end(); ++it1) { + const auto &op_node1 = it1->first; + const auto &op_desc1 = op_node1->GetOpDesc(); + if (op_desc1->HasAttr(ATTR_NAME_CONTROL_FLOW_GROUP)) { + continue; + } + + if (IsUnknownShapeTensor(op_desc1->GetOutputDesc(0))) { + int64_t group_index = op_desc1->GetId(); + GELOGI("Mark %s as unknown shape control flow, group index: %ld", op_desc1->GetName().c_str(), group_index); + MarkForceUnknownShape(op_node1, true, group_index); + for (const auto &n : it1->second) { + MarkForceUnknownShape(n, true, group_index); + } + + for (auto it2 = switch_groups.begin(); it2 != switch_groups.end(); ++it2) { + const auto &op_node2 = it2->first; + const auto &op_desc2 = op_node2->GetOpDesc(); + if (op_desc2->HasAttr(ATTR_NAME_CONTROL_FLOW_GROUP)) { + continue; + } + + if (std::any_of(it2->second.begin(), it2->second.end(), callback)) { + MarkForceUnknownShape(op_node2, true, group_index); + for (const auto &n : it2->second) { + MarkForceUnknownShape(n, true, group_index); + } + } } } } diff --git a/ge/graph/passes/mark_graph_unknown_status_pass.cc b/ge/graph/passes/mark_graph_unknown_status_pass.cc index bf69480a..6aab5478 100644 --- a/ge/graph/passes/mark_graph_unknown_status_pass.cc +++ b/ge/graph/passes/mark_graph_unknown_status_pass.cc @@ -29,7 +29,7 @@ Status MarkGraphUnknownStatusPass::Run(ComputeGraphPtr graph) { bool forced_unknown = false; for (const auto &node : graph->GetDirectNode()) { GE_CHK_GRAPH_STATUS_RET(ge::NodeUtils::GetNodeUnknownShapeStatus(*node, is_unknown_shape), - "Get node[%s] shape status failed!", node->GetName().c_str()); + "Get node[%s] shape status failed!", node->GetName().c_str()); if (is_unknown_shape) { break; } diff --git a/ge/graph/passes/memcpy_addr_async_pass.cc b/ge/graph/passes/memcpy_addr_async_pass.cc index aff89f35..e8e4ebd8 100755 --- a/ge/graph/passes/memcpy_addr_async_pass.cc +++ b/ge/graph/passes/memcpy_addr_async_pass.cc @@ -25,15 +25,15 @@ namespace ge { Status MemcpyAddrAsyncPass::Run(ComputeGraphPtr graph) { GE_CHECK_NOTNULL(graph); - for (const auto &node : graph->GetAllNodes()) { - if (node->GetType() == STREAMSWITCH) { - auto sub_graph = node->GetOwnerComputeGraph(); - if (sub_graph != nullptr && !sub_graph->GetGraphUnknownFlag()) { - GE_CHK_STATUS_RET(AddMemcpyAsyncNode(node), "Add memcpyasync node failed in known subgraph."); + if (graph->GetGraphUnknownFlag()) { + for (const auto &node : graph->GetAllNodes()) { + if (node->GetType() == STREAMSWITCH) { + auto sub_graph = node->GetOwnerComputeGraph(); + if (sub_graph != nullptr && !sub_graph->GetGraphUnknownFlag()) { + GE_CHK_STATUS_RET(AddMemcpyAsyncNode(node), "Add memcpyasync node failed in known subgraph."); + } } } - } - if (graph->GetGraphUnknownFlag()) { GELOGD("Graph[%s] is unknown graph, skip.", graph->GetName().c_str()); return SUCCESS; } diff --git a/ge/graph/passes/merge_to_stream_merge_pass.cc b/ge/graph/passes/merge_to_stream_merge_pass.cc index f3a437a6..4c1ad1ae 100644 --- a/ge/graph/passes/merge_to_stream_merge_pass.cc +++ b/ge/graph/passes/merge_to_stream_merge_pass.cc @@ -84,8 +84,9 @@ Status MergeToStreamMergePass::AddActiveNodes(const ComputeGraphPtr &graph, cons GE_CHK_BOOL_EXEC(node != nullptr, REPORT_INNER_ERROR("E19999", "Param node is nullptr, check invalid"); return FAILED, "Param of pre node is null."); - bool force_unknown = node->GetOpDesc()->HasAttr(ATTR_NAME_FORCE_UNKNOWN_SHAPE); - MarkForceUnknownShape(node, force_unknown); + int64_t group_index = -1; + bool force_unknown = AttrUtils::GetInt(node->GetOpDesc(), ATTR_NAME_CONTROL_FLOW_GROUP, group_index); + MarkForceUnknownShape(node, force_unknown, group_index); for (const InDataAnchorPtr &in_data_anchor : node->GetAllInDataAnchors()) { OutDataAnchorPtr peer_out_anchor = in_data_anchor->GetPeerOutAnchor(); GE_IF_BOOL_EXEC(peer_out_anchor == nullptr, continue); @@ -102,7 +103,7 @@ Status MergeToStreamMergePass::AddActiveNodes(const ComputeGraphPtr &graph, cons GELOGE(FAILED, "SetActiveLabelList for node %s failed.", active_node->GetName().c_str()); return FAILED; } - MarkForceUnknownShape(active_node, force_unknown); + MarkForceUnknownShape(active_node, force_unknown, group_index); } return SUCCESS; diff --git a/ge/graph/passes/multi_batch_clone_pass.cc b/ge/graph/passes/multi_batch_clone_pass.cc index 8d4bcb66..8fc25f9d 100755 --- a/ge/graph/passes/multi_batch_clone_pass.cc +++ b/ge/graph/passes/multi_batch_clone_pass.cc @@ -1020,7 +1020,7 @@ Status MultiBatchClonePass::SetShapeToData(const std::vector &shapes, c if (!IsGetNextType(data)) { if (NodeUtils::UpdateInputShape(*data, kDataInIndex, data_shape) != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Update input desc shape to op:%s(%s) failed, index:%u", - data->GetName().c_str(), data->GetType().c_str(), kDataInIndex); + data->GetName().c_str(), data->GetType().c_str(), kDataInIndex); GELOGE(INTERNAL_ERROR, "Failed to update input shape for data %s", data->GetName().c_str()); return INTERNAL_ERROR; } diff --git a/ge/graph/passes/multi_batch_pass.cc b/ge/graph/passes/multi_batch_pass.cc index eafe982c..2c654801 100644 --- a/ge/graph/passes/multi_batch_pass.cc +++ b/ge/graph/passes/multi_batch_pass.cc @@ -759,7 +759,7 @@ Status MultiBatchPass::AttachStreamLabel(uint32_t batch_idx, const std::string & GELOGD("Attach stream_label %s to node %s.", stream_label.c_str(), cur_desc->GetName().c_str()); if (SetStreamLabel(cur_node, stream_label) != SUCCESS) { REPORT_CALL_ERROR("E19999", "Set stream_label:%s to op:%s(%s) failed", - stream_label.c_str(), cur_node->GetName().c_str(), cur_node->GetType().c_str()); + stream_label.c_str(), cur_node->GetName().c_str(), cur_node->GetType().c_str()); GELOGE(FAILED, "Set stream_label failed, node:%s.", cur_node->GetName().c_str()); return FAILED; } diff --git a/ge/graph/passes/net_output_pass.cc b/ge/graph/passes/net_output_pass.cc index aca7058d..2b27e003 100644 --- a/ge/graph/passes/net_output_pass.cc +++ b/ge/graph/passes/net_output_pass.cc @@ -73,7 +73,7 @@ Status NetOutputPass::GetRetvalOutputInfo(const ge::NodePtr &node, if (iter != targets_.end()) { targets_.erase(iter); targets_.insert(src_node_ptr); - GELOGI("node [%s] is in user def targets, do not output result to user!", node->GetName().c_str()); + GELOGI("Node [%s] is in user def targets, do not output result to user!", node->GetName().c_str()); } is_include_special_node_ = true; return SUCCESS; @@ -105,7 +105,7 @@ Status NetOutputPass::GetOutputNode(const ge::ComputeGraphPtr &graph, std::vecto for (auto &ele : graph->GetGraphOutNodesInfo()) { auto iter = targets_.find(ele.first); if (iter != targets_.end()) { - GELOGI("user set out node [%s] is found in user def targets, out node is prio!", ele.first->GetName().c_str()); + GELOGI("User set out node [%s] is found in user def targets, out node is prior!", ele.first->GetName().c_str()); targets_.erase(iter); } @@ -213,7 +213,7 @@ Status NetOutputPass::UpdateNetOutputDesc(const ge::NodePtr &net_output) { std::vector is_input_const; for (const auto &in_anchor : net_output->GetAllInDataAnchors()) { GE_CHECK_NOTNULL(in_anchor); - uint32_t index = static_cast(in_anchor->GetIdx()); + auto index = static_cast(in_anchor->GetIdx()); if (index >= net_output_desc->GetAllInputsDesc().size()) { REPORT_INNER_ERROR("E19999", "Node:%s(%s) has in_anchor index:%u >= its input desc num:%zu, check invalid", net_output_desc->GetName().c_str(), net_output_desc->GetType().c_str(), index, @@ -369,10 +369,9 @@ Status NetOutputPass::UnLinkDataAnchorOfNetoutput(const ge::ComputeGraphPtr &gra if (!CheckNodeIsInOutputNodes(graph, node)) { ret = in_data_anchor->Unlink(peer_out_anchor); if (ret != SUCCESS) { - REPORT_CALL_ERROR("E19999", - "Op:%s(%s) out index:%d unlink from op:%s(%s) in index:%d failed", - net_out_node->GetName().c_str(), net_out_node->GetType().c_str(), in_data_anchor->GetIdx(), - node->GetName().c_str(), node->GetType().c_str(), peer_out_anchor->GetIdx()); + REPORT_CALL_ERROR("E19999", "Op:%s(%s) out index:%d unlink from op:%s(%s) in index:%d failed", + net_out_node->GetName().c_str(), net_out_node->GetType().c_str(), in_data_anchor->GetIdx(), + node->GetName().c_str(), node->GetType().c_str(), peer_out_anchor->GetIdx()); GELOGE(INTERNAL_ERROR, "Unlink peer_out_anchor fail!"); return ret; } @@ -565,7 +564,7 @@ Status NetOutputPass::AddNetOutputNodeToGraph(const ge::ComputeGraphPtr &graph, GELOGI("[NETOUTPUT PASS] Add net output node succeed"); return SUCCESS; } - GELOGI("[NETOUTPUT PASS] Output node size:%lu.", output_nodes_info.size()); + GELOGI("[NETOUTPUT PASS] Output node size:%zu.", output_nodes_info.size()); if (output_nodes_info.empty()) { // because retval node is contained by output_nodes_info, here means targets is non-empty output_node = graph->AddNode(net_output_desc); diff --git a/ge/graph/passes/next_iteration_pass.cc b/ge/graph/passes/next_iteration_pass.cc index 5f4fc4d0..71b9e621 100644 --- a/ge/graph/passes/next_iteration_pass.cc +++ b/ge/graph/passes/next_iteration_pass.cc @@ -18,6 +18,7 @@ #include "common/ge/ge_util.h" #include "graph/common/omg_util.h" +#include "graph/utils/node_utils.h" using std::string; @@ -203,6 +204,7 @@ Status NextIterationPass::HandleWhileGroup(ComputeGraphPtr &graph) { for (const auto &loop_cond_iter : loop_group_map_) { const LoopCondGroup &loop_group = *loop_cond_iter.second; const std::string &cond_name = loop_cond_iter.second->loop_cond->GetName(); + const int64_t group_index = loop_group.loop_cond->GetOpDesc()->GetId(); GELOGI("Handle while group, LoopCond node: %s.", cond_name.c_str()); // Create Active node, Enter->Active->Merge, NextIteration->Active->Merge @@ -223,7 +225,7 @@ Status NextIterationPass::HandleWhileGroup(ComputeGraphPtr &graph) { enter_active->GetName().c_str()); return INTERNAL_ERROR; } - MarkForceUnknownShape(enter_node, loop_group.is_unknown_shape); + MarkForceUnknownShape(enter_node, loop_group.is_unknown_shape, group_index); } for (const auto &pair : loop_cond_iter.second->merge_next_pairs) { @@ -253,8 +255,8 @@ Status NextIterationPass::HandleWhileGroup(ComputeGraphPtr &graph) { return INTERNAL_ERROR; } - MarkForceUnknownShape(next_node, loop_group.is_unknown_shape); - MarkForceUnknownShape(merge_node, loop_group.is_unknown_shape); + MarkForceUnknownShape(next_node, loop_group.is_unknown_shape, group_index); + MarkForceUnknownShape(merge_node, loop_group.is_unknown_shape, group_index); } if ((SetActiveLabelList(enter_active, {cond_name}) != SUCCESS) || @@ -263,10 +265,10 @@ Status NextIterationPass::HandleWhileGroup(ComputeGraphPtr &graph) { return INTERNAL_ERROR; } - MarkForceUnknownShape(loop_group.loop_cond, loop_group.is_unknown_shape); - MarkForceUnknownShape(enter_active, loop_group.is_unknown_shape); - MarkForceUnknownShape(next_active, loop_group.is_unknown_shape); - HandleSwitchExitNodes(loop_group); + MarkForceUnknownShape(loop_group.loop_cond, loop_group.is_unknown_shape, group_index); + MarkForceUnknownShape(enter_active, loop_group.is_unknown_shape, group_index); + MarkForceUnknownShape(next_active, loop_group.is_unknown_shape, group_index); + HandleSwitchExitNodes(loop_group, group_index); } return SUCCESS; @@ -275,20 +277,21 @@ Status NextIterationPass::HandleWhileGroup(ComputeGraphPtr &graph) { /// /// @brief Mark force unknown for Exit node /// @param [in] group of LoopCond +/// @param [in] index of LoopCond Node /// @return void /// -void NextIterationPass::HandleSwitchExitNodes(const LoopCondGroup &loop_group) { +void NextIterationPass::HandleSwitchExitNodes(const LoopCondGroup &loop_group, int64_t group_index) { if (!loop_group.is_unknown_shape) { return; } for (const auto &switch_node : loop_group.switch_nodes) { - MarkForceUnknownShape(switch_node, loop_group.is_unknown_shape); + MarkForceUnknownShape(switch_node, loop_group.is_unknown_shape, group_index); for (const auto &node : switch_node->GetOutDataNodes()) { std::string node_type; (void)GetOriginalType(node, node_type); - if (node_type == EXIT || node_type == REFEXIT) { - MarkForceUnknownShape(node, loop_group.is_unknown_shape); + if (kExitOpTypes.count(node_type) > 0) { + MarkForceUnknownShape(node, loop_group.is_unknown_shape, group_index); } } } @@ -351,7 +354,7 @@ Status NextIterationPass::BreakNextIteration(const NodePtr &next_node, NodePtr & merge_node->GetName().c_str()); return INTERNAL_ERROR; } - if (SetNextIteration(merge_node, next_node->GetName()) != SUCCESS) { + if (SetNextIteration(merge_node, next_node) != SUCCESS) { REPORT_CALL_ERROR("E19999", "Set attr NEXT_ITERATION value:%s to node:%s(%s) failed", next_node->GetName().c_str(), merge_node->GetName().c_str(), merge_node->GetType().c_str()); GELOGE(INTERNAL_ERROR, "Set attr NEXT_ITERATION for node %s failed.", merge_node->GetName().c_str()); diff --git a/ge/graph/passes/next_iteration_pass.h b/ge/graph/passes/next_iteration_pass.h index e8786516..b6a0846d 100755 --- a/ge/graph/passes/next_iteration_pass.h +++ b/ge/graph/passes/next_iteration_pass.h @@ -96,9 +96,10 @@ class NextIterationPass : public GraphPass { /// /// @brief Mark force unknown for Exit node /// @param [in] group of LoopCond + /// @param [in] index of LoopCond Node /// @return void /// - void HandleSwitchExitNodes(const LoopCondGroup &loop_group); + void HandleSwitchExitNodes(const LoopCondGroup &loop_group, int64_t group_index); // map std::unordered_map loop_group_map_; diff --git a/ge/graph/passes/pass_utils.cc b/ge/graph/passes/pass_utils.cc index db379433..1fc675b1 100644 --- a/ge/graph/passes/pass_utils.cc +++ b/ge/graph/passes/pass_utils.cc @@ -170,7 +170,7 @@ Status PassUtils::SetOutNodeWeight(const OutDataAnchorPtr &out_data_anchor, cons // restore control inputs to dynamically added constant ops, if any for (const auto &src_out_control_anchor : src_out_control_anchors) { GE_CHK_GRAPH_STATUS_RET(GraphUtils::AddEdge(src_out_control_anchor, dynamic_const_node->GetInControlAnchor()), - "add edge failed"); + "add edge failed"); } } diff --git a/ge/graph/passes/subgraph_pass.cc b/ge/graph/passes/subgraph_pass.cc index b931eea8..401dee54 100755 --- a/ge/graph/passes/subgraph_pass.cc +++ b/ge/graph/passes/subgraph_pass.cc @@ -464,8 +464,8 @@ Status SubgraphPass::InsertMemcpyNode(const ComputeGraphPtr &graph, const OutDat GE_CHECK_NOTNULL(out_anchor); NodePtr in_node = out_anchor->GetOwnerNode(); OpDescBuilder op_desc_builder(name, IDENTITY); - OpDescPtr op_desc = op_desc_builder.AddInput("x", in_node->GetOpDesc()->GetOutputDesc(0)) - .AddOutput("y", in_node->GetOpDesc()->GetOutputDesc(0)) + OpDescPtr op_desc = op_desc_builder.AddInput("x", in_node->GetOpDesc()->GetOutputDesc(out_anchor->GetIdx())) + .AddOutput("y", in_node->GetOpDesc()->GetOutputDesc(out_anchor->GetIdx())) .Build(); (void)AttrUtils::SetBool(op_desc, ATTR_NO_NEED_CONSTANT_FOLDING, false); (void)AttrUtils::SetBool(op_desc, ATTR_NAME_CANNOT_BE_DELETED, true); diff --git a/ge/graph/passes/switch_data_edges_bypass.cc b/ge/graph/passes/switch_data_edges_bypass.cc index 6a925ae3..5c324238 100644 --- a/ge/graph/passes/switch_data_edges_bypass.cc +++ b/ge/graph/passes/switch_data_edges_bypass.cc @@ -51,7 +51,7 @@ std::vector> GetOutDataNodesByIndex(const No auto out_anchor = node->GetOutDataAnchor(index); if (out_anchor == nullptr) { REPORT_INNER_ERROR("E19999", "Node:%s(%s) has no index:%d out data anchor, check invalid", - node->GetName().c_str(), node->GetType().c_str(), index); + node->GetName().c_str(), node->GetType().c_str(), index); GELOGE(PARAM_INVALID, "Failed to get out data nodes of index %d from node %s, the anchor does not exists", index, node->GetName().c_str()); return {}; diff --git a/ge/graph/passes/switch_to_stream_switch_pass.cc b/ge/graph/passes/switch_to_stream_switch_pass.cc index 949fff41..af87dafa 100644 --- a/ge/graph/passes/switch_to_stream_switch_pass.cc +++ b/ge/graph/passes/switch_to_stream_switch_pass.cc @@ -369,7 +369,9 @@ NodePtr SwitchToStreamSwitchPass::CreateStreamSwitchNode(const ComputeGraphPtr & GE_CHK_STATUS(GraphUtils::AddEdge(peer_cond_anchor, stream_switch->GetInDataAnchor(0)), "StreamSwitch node add cond edge failed."); - MarkForceUnknownShape(stream_switch, switch_node->GetOpDesc()->HasAttr(ATTR_NAME_FORCE_UNKNOWN_SHAPE)); + int64_t group_index = -1; + bool force_unknown = AttrUtils::GetInt(switch_node->GetOpDesc(), ATTR_NAME_CONTROL_FLOW_GROUP, group_index); + MarkForceUnknownShape(stream_switch, force_unknown, group_index); return stream_switch; } @@ -488,11 +490,12 @@ Status SwitchToStreamSwitchPass::CombineSwitchNode(const ComputeGraphPtr &graph) return FAILED; } - std::function callback = [](const NodePtr &n) { - return n->GetOpDesc()->HasAttr(ATTR_NAME_FORCE_UNKNOWN_SHAPE); + int64_t group_index = -1; + std::function callback = [&group_index](const NodePtr &n) { + return AttrUtils::GetInt(n->GetOpDesc(), ATTR_NAME_CONTROL_FLOW_GROUP, group_index); }; bool is_unknown_shape = std::any_of(same_cond_switch.begin(), same_cond_switch.end(), callback); - MarkForceUnknownShape(active_node, is_unknown_shape); + MarkForceUnknownShape(active_node, is_unknown_shape, group_index); const std::string &cond_group = cond_node->GetName(); for (uint32_t i = 0; i < SWITCH_OUTPUT_NUM; ++i) { @@ -522,7 +525,7 @@ Status SwitchToStreamSwitchPass::CombineSwitchNode(const ComputeGraphPtr &graph) GE_CHK_STATUS(GraphUtils::AddEdge(cast_node->GetOutDataAnchor(0), stream_switch->GetInDataAnchor(0)), "Cast add data edge failed."); - MarkForceUnknownShape(stream_switch, is_unknown_shape); + MarkForceUnknownShape(stream_switch, is_unknown_shape, group_index); for (const NodePtr &node : switch_list) { GE_IF_BOOL_EXEC(node != stream_switch, { GE_CHK_STATUS(GraphUtils::RemoveEdge(peer_cond_anchor, node->GetInDataAnchor(0)), @@ -562,13 +565,6 @@ NodePtr SwitchToStreamSwitchPass::CreateActiveNode(const ComputeGraphPtr &graph, op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str()); return nullptr, "Create StreamActive node failed."); - GE_IF_BOOL_EXEC(GraphUtils::AddEdge(node->GetOutControlAnchor(), active_node->GetInControlAnchor()) != SUCCESS, - REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed", - node->GetName().c_str(), node->GetType().c_str(), - active_node->GetName().c_str(), active_node->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "add edge failed"); - return nullptr); - GE_IF_BOOL_EXEC(SetSwitchBranchNodeLabel(active_node, node_name) != SUCCESS, REPORT_CALL_ERROR("E19999", "Set switch branch node label:%s to node:%s(%s) failed", node_name.c_str(), active_node->GetName().c_str(), active_node->GetType().c_str()); diff --git a/ge/graph/passes/transop_without_reshape_fusion_pass.cc b/ge/graph/passes/transop_without_reshape_fusion_pass.cc index 00896235..cd16258a 100644 --- a/ge/graph/passes/transop_without_reshape_fusion_pass.cc +++ b/ge/graph/passes/transop_without_reshape_fusion_pass.cc @@ -1077,9 +1077,9 @@ graphStatus TransOpWithoutReshapeFusionPass::RelinkControlEdge(const int index, peer_in_anchor->GetOwnerNode()->GetName().c_str()); if (GraphUtils::AddEdge(new_trans_nodes.back()->GetOutControlAnchor(), peer_in_anchor) != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed", - new_trans_nodes.back()->GetName().c_str(), new_trans_nodes.back()->GetType().c_str(), - peer_in_anchor->GetOwnerNode()->GetName().c_str(), - peer_in_anchor->GetOwnerNode()->GetType().c_str()); + new_trans_nodes.back()->GetName().c_str(), new_trans_nodes.back()->GetType().c_str(), + peer_in_anchor->GetOwnerNode()->GetName().c_str(), + peer_in_anchor->GetOwnerNode()->GetType().c_str()); return GRAPH_FAILED; } } @@ -1103,9 +1103,9 @@ graphStatus TransOpWithoutReshapeFusionPass::RelinkControlEdge(const int index, peer_in_anchor->GetOwnerNode()->GetName().c_str()); if (GraphUtils::AddEdge(new_trans_nodes.back()->GetOutControlAnchor(), peer_in_anchor) != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Add control edge between op:%s(%s) and op:%s(%s) failed", - new_trans_nodes.back()->GetName().c_str(), new_trans_nodes.back()->GetType().c_str(), - peer_in_anchor->GetOwnerNode()->GetName().c_str(), - peer_in_anchor->GetOwnerNode()->GetType().c_str()); + new_trans_nodes.back()->GetName().c_str(), new_trans_nodes.back()->GetType().c_str(), + peer_in_anchor->GetOwnerNode()->GetName().c_str(), + peer_in_anchor->GetOwnerNode()->GetType().c_str()); return GRAPH_FAILED; } } diff --git a/ge/graph/passes/variable_op_pass.cc b/ge/graph/passes/variable_op_pass.cc index c605d305..3b3328a3 100644 --- a/ge/graph/passes/variable_op_pass.cc +++ b/ge/graph/passes/variable_op_pass.cc @@ -87,10 +87,10 @@ Status ByPassTransNode(NodePtr &trans_node, NodePtr &ref_node) { ret = GraphUtils::AddEdge(prev_trans_node_out_anchor, ref_in_anchor); if (ret != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(index:%d) and op:%s(%s)(index:0) failed", - prev_trans_node_out_anchor->GetOwnerNode()->GetName().c_str(), - prev_trans_node_out_anchor->GetOwnerNode()->GetType().c_str(), - prev_trans_node_out_anchor->GetIdx(), - ref_node->GetName().c_str(), ref_node->GetType().c_str()); + prev_trans_node_out_anchor->GetOwnerNode()->GetName().c_str(), + prev_trans_node_out_anchor->GetOwnerNode()->GetType().c_str(), + prev_trans_node_out_anchor->GetIdx(), + ref_node->GetName().c_str(), ref_node->GetType().c_str()); GELOGE(INTERNAL_ERROR, "Failed to add edge between ref node %s " "and the prev node of trans node %s", diff --git a/ge/graph/preprocess/graph_preprocess.cc b/ge/graph/preprocess/graph_preprocess.cc index 4e9046e4..e1921f29 100644 --- a/ge/graph/preprocess/graph_preprocess.cc +++ b/ge/graph/preprocess/graph_preprocess.cc @@ -74,6 +74,7 @@ #include "graph/passes/unused_const_pass.h" #include "graph/passes/var_is_initialized_op_pass.h" #include "graph/passes/variable_prepare_op_pass.h" +#include "graph/passes/mark_force_unknown_for_cond_pass.h" #include "graph/preprocess/insert_op/util_insert_aipp_op.h" #include "graph/utils/type_utils.h" #include "inc/pass_manager.h" @@ -101,7 +102,7 @@ OpDescPtr CreateTensorShape(const GeTensorDesc &data_tensor) { GeTensorPtr tensor = MakeShared(); if (tensor == nullptr) { REPORT_CALL_ERROR("E19999", "New GeTensor failed"); - GELOGE(INTERNAL_ERROR, "Create shared ptr for GeTensor failed"); + GELOGE(INTERNAL_ERROR, "[New][GeTensor] failed"); return nullptr; } tensor->MutableTensorDesc().SetDataType(DT_INT32); @@ -113,7 +114,7 @@ OpDescPtr CreateTensorShape(const GeTensorDesc &data_tensor) { int32_t dst_shape = 1; if (tensor->SetData(reinterpret_cast(&dst_shape), sizeof(int32_t)) != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Set data to tensor failed"); - GELOGE(INTERNAL_ERROR, "tensor set data failed"); + GELOGE(INTERNAL_ERROR, "[Set][Data] to tensor failed"); return nullptr; } } else { @@ -121,7 +122,7 @@ OpDescPtr CreateTensorShape(const GeTensorDesc &data_tensor) { unique_ptr dst_shape(new (std::nothrow) int32_t[dim_cnt]()); if (dst_shape == nullptr) { REPORT_CALL_ERROR("E19999", "Malloc buffer failed, size:%zu", dim_cnt); - GELOGE(INTERNAL_ERROR, "Create unique ptr failed"); + GELOGE(INTERNAL_ERROR, "[Malloc][Buffer] failed, size:%zu", dim_cnt); return nullptr; } for (int64_t i = 0; i < dim_cnt; ++i) { @@ -131,7 +132,7 @@ OpDescPtr CreateTensorShape(const GeTensorDesc &data_tensor) { GE_IF_BOOL_EXEC( tensor->SetData(reinterpret_cast(dst_shape.get()), dim_cnt * sizeof(int32_t)) != GRAPH_SUCCESS, REPORT_CALL_ERROR("E19999", "Set data to tensor failed"); - GELOGE(INTERNAL_ERROR, "tensor set data failed"); + GELOGE(INTERNAL_ERROR, "[Set][Data] to tensor failed"); return nullptr;) } @@ -179,14 +180,16 @@ NodePtr CreateTransNode(const std::string &name, const std::string &node_type, c if (node == nullptr) { REPORT_INNER_ERROR("E19999", "Param node is nullptr, trans_name:%s, trans_type:%s, check invalid", name.c_str(), node_type.c_str()); - GELOGE(PARAM_INVALID, "node is null."); + GELOGE(PARAM_INVALID, "[Check][Param] Param node is nullptr, trans_name:%s, trans_type:%s", + name.c_str(), node_type.c_str()); return nullptr; } auto graph = node->GetOwnerComputeGraph(); if (graph == nullptr) { REPORT_INNER_ERROR("E19999", "Owner graph in node is nullptr, trans_name:%s, trans_type:%s, check invalid", name.c_str(), node_type.c_str()); - GELOGE(PARAM_INVALID, "Owner graph is null, node name:%s.", node->GetName().c_str()); + GELOGE(PARAM_INVALID, "[Get][OwnerGraph] in node is nullptr, trans_name:%s, trans_type:%s", + name.c_str(), node_type.c_str()); return nullptr; } @@ -195,14 +198,15 @@ NodePtr CreateTransNode(const std::string &name, const std::string &node_type, c ErrorManager::GetInstance().ATCReportErrMessage( "E19025", {"situation", "reason"}, {"The trans node type[" + node_type + "]", "it must be " + TransOpUtil::TransopMapToString()}); - GELOGE(INTERNAL_ERROR, "The trans node type %s does not exists", node_type.c_str()); + GELOGE(INTERNAL_ERROR, "[Check][Param] The trans node type %s does not exists", node_type.c_str()); return nullptr; } OpDescPtr op_desc = MakeShared(name, node_type); if (op_desc == nullptr) { - REPORT_CALL_ERROR("E19999", "New OpDesc failed, trans_name:%s, trans_type:%s,", + REPORT_CALL_ERROR("E19999", "New OpDesc failed, trans_name:%s, trans_type:%s", name.c_str(), node_type.c_str()); - GELOGE(INTERNAL_ERROR, "Create shared ptr for OpDesc failed"); + GELOGE(INTERNAL_ERROR, "[New][OpDesc] failed, trans_name:%s, trans_type:%s", + name.c_str(), node_type.c_str()); return nullptr; } @@ -216,14 +220,16 @@ NodePtr CreateTransNode(const std::string &name, const std::string &node_type, c if (ret != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Add input desc into op:%s(%s) failed", op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to add input desc when create node %s type %s", name.c_str(), node_type.c_str()); + GELOGE(INTERNAL_ERROR, "[Add][InputDesc] into op:%s(%s) failed", + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return nullptr; } ret = op_desc->AddOutputDesc(output); if (ret != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Add output desc into op:%s(%s) failed", op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to add output desc when create node %s type %s", name.c_str(), node_type.c_str()); + GELOGE(INTERNAL_ERROR, "[Add][OutputDesc] into op:%s(%s) failed", + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return nullptr; } @@ -233,15 +239,16 @@ NodePtr CreateTransNode(const std::string &name, const std::string &node_type, c if (node_type == RESHAPE) { auto shape_desc = CreateTensorShape(output); if (shape_desc == nullptr) { - GELOGE(INTERNAL_ERROR, "Failed to add shape for reshape %s, can not create the shape input", + GELOGE(INTERNAL_ERROR, "[Create][TensorShape] Failed to add shape for reshape %s", node->GetName().c_str()); return nullptr; } ret = op_desc->AddInputDesc(shape_desc->GetOutputDesc(0)); if (ret != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Add input desc into op:%s(%s) failed", - op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to add the first input for reshape %s", name.c_str()); + op_desc->GetName().c_str(), op_desc->GetType().c_str()); + GELOGE(INTERNAL_ERROR, "[Add][InputDesc] into op:%s(%s) failed", + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return nullptr; } @@ -250,7 +257,8 @@ NodePtr CreateTransNode(const std::string &name, const std::string &node_type, c REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s failed", shape_desc->GetName().c_str(), shape_desc->GetType().c_str(), graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to add shape node for reshape %s, can not add the shape to graph", name.c_str()); + GELOGE(INTERNAL_ERROR, "[Add][Node] %s(%s) to graph:%s failed", + shape_desc->GetName().c_str(), shape_desc->GetType().c_str(), graph->GetName().c_str()); return nullptr; } } @@ -260,16 +268,19 @@ NodePtr CreateTransNode(const std::string &name, const std::string &node_type, c REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s failed", op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to add trans node %s to graph", name.c_str()); + GELOGE(INTERNAL_ERROR, "[Add][Node] %s(%s) to graph:%s failed", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), graph->GetName().c_str()); return nullptr; } if (node_type == RESHAPE) { if (GraphUtils::AddEdge(shape_node->GetOutDataAnchor(0), trans_node->GetInDataAnchor(1)) != GRAPH_SUCCESS) { - REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(out_index:0) and op:%s(%s)(in_index:0) failed", + REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(out_index:0) and op:%s(%s)(in_index:1) failed", shape_node->GetName().c_str(), shape_node->GetType().c_str(), trans_node->GetName().c_str(), trans_node->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to add shape node for reshape %s, can not add the edge", name.c_str()); + GELOGE(INTERNAL_ERROR, "[Add][Edge] between op:%s(%s)(out_index:0) and op:%s(%s)(in_index:1) failed", + shape_node->GetName().c_str(), shape_node->GetType().c_str(), + trans_node->GetName().c_str(), trans_node->GetType().c_str()); return nullptr; } } @@ -290,8 +301,9 @@ Status RecoverOneTransNodeForVar(const std::string &name, const TransNodeInfo &t REPORT_CALL_ERROR("E19999", "Replace out anchors of node:%s(%s) by node:%s(%s) failed", node->GetName().c_str(), node->GetType().c_str(), trans_node->GetName().c_str(), trans_node->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to replace out anchors when recover trans node for %s type %s", - node->GetName().c_str(), node->GetType().c_str()); + GELOGE(INTERNAL_ERROR, "[Replace][OutAnchors] of node:%s(%s) by node:%s(%s) failed", + node->GetName().c_str(), node->GetType().c_str(), + trans_node->GetName().c_str(), trans_node->GetType().c_str()); return INTERNAL_ERROR; } @@ -300,8 +312,9 @@ Status RecoverOneTransNodeForVar(const std::string &name, const TransNodeInfo &t REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(out_index:0) and op:%s(%s)(in_index:0) failed", node->GetName().c_str(), node->GetType().c_str(), trans_node->GetName().c_str(), trans_node->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to connect node %s to trans node %s", node->GetName().c_str(), - trans_node->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Add][Edge] between op:%s(%s)(out_index:0) and op:%s(%s)(in_index:0) failed", + node->GetName().c_str(), node->GetType().c_str(), + trans_node->GetName().c_str(), trans_node->GetType().c_str()); return INTERNAL_ERROR; } @@ -310,8 +323,9 @@ Status RecoverOneTransNodeForVar(const std::string &name, const TransNodeInfo &t REPORT_CALL_ERROR("E19999", "Move out control edges from node:%s(%s) to node:%s(%s) failed", node->GetName().c_str(), node->GetType().c_str(), trans_node->GetName().c_str(), trans_node->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to move out control edges from %s to %s when recover trans node.", - node->GetName().c_str(), trans_node->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[MoveOut][ControlEdges] from node:%s(%s) to node:%s(%s) failed", + node->GetName().c_str(), node->GetType().c_str(), + trans_node->GetName().c_str(), trans_node->GetType().c_str()); return INTERNAL_ERROR; } @@ -331,8 +345,9 @@ Status RecoverOneTransNodeForVarRef(const std::string &name, const TransNodeInfo REPORT_CALL_ERROR("E19999", "Replace out anchors of node:%s(%s) by node:%s(%s) failed", node->GetName().c_str(), node->GetType().c_str(), trans_node->GetName().c_str(), trans_node->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to replace int anchors when recover trans node for %s type %s", - node->GetName().c_str(), node->GetType().c_str()); + GELOGE(INTERNAL_ERROR, "[Replace][OutAnchors] of node:%s(%s) by node:%s(%s) failed", + node->GetName().c_str(), node->GetType().c_str(), + trans_node->GetName().c_str(), trans_node->GetType().c_str()); return INTERNAL_ERROR; } @@ -341,18 +356,20 @@ Status RecoverOneTransNodeForVarRef(const std::string &name, const TransNodeInfo REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(out_index:0) and op:%s(%s)(in_index:0) failed", trans_node->GetName().c_str(), trans_node->GetType().c_str(), node->GetName().c_str(), node->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to connect trans node %s to node %s", trans_node->GetName().c_str(), - node->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Add][Edge] between op:%s(%s)(out_index:0) and op:%s(%s)(in_index:0) failed", + trans_node->GetName().c_str(), trans_node->GetType().c_str(), + node->GetName().c_str(), node->GetType().c_str()); return INTERNAL_ERROR; } ret = GraphUtils::MoveInCtrlEdges(node, trans_node); if (ret != GRAPH_SUCCESS) { - REPORT_CALL_ERROR("E19999", "Move out control edges from node:%s(%s) to node:%s(%s) failed", + REPORT_CALL_ERROR("E19999", "Move in control edges from node:%s(%s) to node:%s(%s) failed", node->GetName().c_str(), node->GetType().c_str(), trans_node->GetName().c_str(), trans_node->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to move int control edges from %s to %s when recover trans node.", - node->GetName().c_str(), trans_node->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[MoveIn][CtrlEdges] from node:%s(%s) to node:%s(%s) failed", + node->GetName().c_str(), node->GetType().c_str(), + trans_node->GetName().c_str(), trans_node->GetType().c_str()); return INTERNAL_ERROR; } @@ -373,7 +390,8 @@ Status UpdateVarFormats(const NodePtr &var, const GeTensorDesc &tensor_desc) { GE_IF_BOOL_EXEC(var->GetOpDesc()->UpdateOutputDesc(0, output_desc) != GRAPH_SUCCESS, REPORT_CALL_ERROR("E19999", "Update output desc of node:%s(%s) failed, index:0,", var->GetName().c_str(), var->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "UpdateOutputDesc failed"); + GELOGE(INTERNAL_ERROR, "[Update][OutputDesc] of node:%s(%s) failed, index:0,", + var->GetName().c_str(), var->GetType().c_str()); return INTERNAL_ERROR;); } @@ -388,7 +406,8 @@ Status UpdateVarFormats(const NodePtr &var, const GeTensorDesc &tensor_desc) { GE_IF_BOOL_EXEC(var->GetOpDesc()->UpdateInputDesc(0, desc) != GRAPH_SUCCESS, REPORT_CALL_ERROR("E19999", "Update input desc of node:%s(%s) failed, index:0,", var->GetName().c_str(), var->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "UpdateInputDesc failed"); + GELOGE(INTERNAL_ERROR, "[Update][InputDesc] of node:%s(%s) failed, index:0,", + var->GetName().c_str(), var->GetType().c_str()); return INTERNAL_ERROR;) } return SUCCESS; @@ -404,7 +423,7 @@ Status RecoverTransRoadForVar(const NodePtr &var, const VarTransRoad &road) { if (ret != SUCCESS) { ErrorManager::GetInstance().ATCReportErrMessage( "E15001", {"variable", "index", "type"}, {var->GetName(), std::to_string(index), iter->node_type}); - GELOGE(INTERNAL_ERROR, "Failed to recover trans node for variable %s, index %d, type %s", var->GetName().c_str(), + GELOGE(INTERNAL_ERROR, "[Recover][TransNode] for variable %s, index %d, type %s", var->GetName().c_str(), index, iter->node_type.c_str()); return INTERNAL_ERROR; } @@ -418,7 +437,8 @@ Status RecoverTransRoadForVar(const NodePtr &var, const VarTransRoad &road) { if (status != ge::SUCCESS) { REPORT_CALL_ERROR("E19999", "Set stream_label:%s to op:%s(%s) failed", stream_label.c_str(), last_node->GetName().c_str(), last_node->GetType().c_str()); - GELOGE(status, "Set stream label failed."); + GELOGE(status, "[Set][StreamLabel] %s to op:%s(%s) failed.", + stream_label.c_str(), last_node->GetName().c_str(), last_node->GetType().c_str()); return status; } } @@ -426,7 +446,9 @@ Status RecoverTransRoadForVar(const NodePtr &var, const VarTransRoad &road) { REPORT_CALL_ERROR("E19999", "Set Attr:%s to node:%s(%s) failed", ge::ATTR_INSERTED_BY_GE.c_str(), last_node->GetName().c_str(), last_node->GetType().c_str()); - return INTERNAL_ERROR, "Set attr ATTR_INSERTED_BY_GE failed."); + return INTERNAL_ERROR, + "[Set][Attr] %s to node:%s(%s) failed", ge::ATTR_INSERTED_BY_GE.c_str(), + last_node->GetName().c_str(), last_node->GetType().c_str()); GELOGD("Recover trans node %s type %s success", trans_name.c_str(), iter->node_type.c_str()); } if (road.empty()) { @@ -447,7 +469,7 @@ Status RecoverTransRoadForVarRef(const std::set &nodes, const VarTransR if (ret != SUCCESS) { ErrorManager::GetInstance().ATCReportErrMessage( "E15001", {"variable", "index", "type"}, {var->GetName(), std::to_string(index), iter->node_type}); - GELOGE(INTERNAL_ERROR, "Failed to recover trans node for variable %s, index %d, type %s", + GELOGE(INTERNAL_ERROR, "[Recover][TransNode] for variable %s failed, index %d, type %s", var->GetName().c_str(), index, iter->node_type.c_str()); return INTERNAL_ERROR; } @@ -461,7 +483,8 @@ Status RecoverTransRoadForVarRef(const std::set &nodes, const VarTransR if (status != ge::SUCCESS) { REPORT_CALL_ERROR("E19999", "Set stream_label:%s to op:%s(%s) failed", stream_label.c_str(), last_node->GetName().c_str(), last_node->GetType().c_str()); - GELOGE(status, "Set stream label failed."); + GELOGE(status, "[Set][StreamLabel] %s to op:%s(%s) failed.", + stream_label.c_str(), last_node->GetName().c_str(), last_node->GetType().c_str()); return status; } } @@ -470,7 +493,9 @@ Status RecoverTransRoadForVarRef(const std::set &nodes, const VarTransR REPORT_CALL_ERROR("E19999", "Set Attr:%s of node:%s(%s) failed", ge::ATTR_INSERTED_BY_GE.c_str(), last_node->GetName().c_str(), last_node->GetType().c_str()); - return INTERNAL_ERROR, "Set attr ATTR_INSERTED_BY_GE failed."); + return INTERNAL_ERROR, + "[Set][Attr] %s of node:%s(%s) failed", ge::ATTR_INSERTED_BY_GE.c_str(), + last_node->GetName().c_str(), last_node->GetType().c_str()); } if (!(road.empty()) && (UpdateVarFormats(var, road.rbegin()->output) != SUCCESS)) { return INTERNAL_ERROR; @@ -486,7 +511,7 @@ VarNamesToRefs CollectVarNamesToRefs(const ComputeGraphPtr &graph) { std::string var_name; if (graph == nullptr) { REPORT_INNER_ERROR("E19999", "Param graph is nullptr, check invalid"); - GELOGE(PARAM_INVALID, "graph is null."); + GELOGE(PARAM_INVALID, "[Check][Param] graph is nullptr."); return names_to_refs; } for (auto &node : graph->GetAllNodes()) { @@ -505,13 +530,13 @@ Status TransferShape2NC1HWC0(Format src_format, const std::vector &src_ if (src_format == FORMAT_NCHW) { formats::FormatTransferNchwNc1hwc0 transfer; if (transfer.TransShape(src_format, src_shape, dt, dst_format, dst_shape) != SUCCESS) { - GELOGE(INTERNAL_ERROR, "TransShape failed"); + GELOGE(INTERNAL_ERROR, "[Trans][Shape] failed"); return FAILED; } } else if (src_format == FORMAT_NHWC) { formats::FormatTransferNhwcNc1hwc0 transfer; if (transfer.TransShape(src_format, src_shape, dt, dst_format, dst_shape) != SUCCESS) { - GELOGE(INTERNAL_ERROR, "TransShape failed"); + GELOGE(INTERNAL_ERROR, "[Trans][Shape] failed"); return FAILED; } } @@ -531,7 +556,8 @@ Status ModifyInputFormatAndShape(NodePtr &node_ptr) { if (TransferShape2NC1HWC0(old_format, old_shape, dt, FORMAT_NC1HWC0, dst_shape_dims) != SUCCESS) { REPORT_CALL_ERROR("E19999", "Transfer shape to NC1HWC0 failed, op:%s(%s),", op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Trans shape failed"); + GELOGE(INTERNAL_ERROR, "[Transfer][Shape] to NC1HWC0 failed, op:%s(%s),", + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return FAILED; } @@ -548,7 +574,8 @@ Status ModifyInputFormatAndShape(NodePtr &node_ptr) { if (graph_status != ge::GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Get output tensor size failed, op:%s(%s), index:0", op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(graph_status, "GetTensorSizeInBytes failed!"); + GELOGE(graph_status, "[Get][TensorSize] In Bytes failed, op:%s(%s), index:0", + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return FAILED; } ge::TensorUtils::SetSize(*output, size); @@ -564,7 +591,7 @@ Status ModifyFormatAndShapeForSingleTensor(const GeTensorDescPtr &input_output) ge::DataType dt = input_output->GetDataType(); std::vector dst_shape_dims; if (TransferShape2NC1HWC0(old_format, old_shape, dt, FORMAT_NC1HWC0, dst_shape_dims) != SUCCESS) { - GELOGE(INTERNAL_ERROR, "Trans shape failed"); + GELOGE(INTERNAL_ERROR, "[Trans][Shape] to NC1HWC0 failed"); return FAILED; } input_output->SetFormat(FORMAT_NC1HWC0); @@ -594,7 +621,8 @@ Status ModifyDataNetOutputFormatAndShape(OpDescPtr &op_desc, uint32_t index, For if (graph_status != ge::GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Get output tensor size failed, op:%s(%s), index:%u", op_desc->GetName().c_str(), op_desc->GetType().c_str(), index); - GELOGE(graph_status, "GetTensorSizeInBytes failed!"); + GELOGE(graph_status, "[Get][TensorSize] In Bytes failed, op:%s(%s), index:%u", + op_desc->GetName().c_str(), op_desc->GetType().c_str(), index); return FAILED; } ge::TensorUtils::SetSize(*input, size); @@ -617,7 +645,7 @@ Status CheckIfDynamicBatchScene(NodePtr &data_node, bool &is_dynamic_batch, Node if (related_node_name.empty()) { ErrorManager::GetInstance().ATCReportErrMessage( "E15002", {"opname", "value", "reason"}, {data_node->GetName(), "flag", "but the value is empty"}); - GELOGE(INTERNAL_ERROR, "The data node %s has switchn node flag, but the value is empty", + GELOGE(INTERNAL_ERROR, "[Check][Param] The data node %s has switchn node flag, but the value is empty", data_node->GetName().c_str()); return INTERNAL_ERROR; } @@ -635,7 +663,7 @@ Status CheckIfDynamicBatchScene(NodePtr &data_node, bool &is_dynamic_batch, Node ErrorManager::GetInstance().ATCReportErrMessage( "E15002", {"opname", "value", "reason"}, {data_node->GetName(), related_node_name, "but can not find it on the graph"}); - GELOGE(INTERNAL_ERROR, "The data node %s has switchn node %s, but can not find it on the graph", + GELOGE(INTERNAL_ERROR, "[Check][Param] The data node %s has switchn node %s, but can not find it on the graph", data_node->GetName().c_str(), related_node_name.c_str()); return INTERNAL_ERROR; } @@ -782,15 +810,15 @@ Status UpdateSubgraphDataOfCase(NodePtr &mbatch_node, DataType &dt_set, int32_t Status ProcessMbatchScene(NodePtr &mbatch_node, DataType &dt_set, int32_t index) { GELOGI("The node [%s] dtype set fp16.", mbatch_node->GetName().c_str()); if (UpdateInputOutputDataType(mbatch_node, dt_set, index) != SUCCESS) { - GELOGE(FAILED, "Update input and output data type of node[name: %s, type: %s] to %s failed.", + GELOGE(FAILED, "[Update][InputOutputDataType] of node[name: %s, type: %s] to %s failed.", mbatch_node->GetName().c_str(), mbatch_node->GetType().c_str(), TypeUtils::DataTypeToSerialString(dt_set).c_str()); return FAILED; } if (UpdateSubgraphDataOfCase(mbatch_node, dt_set, index) != SUCCESS) { - GELOGE(FAILED, "Update input and output data type of Data node[parent_node_index: %d] in subgraphs of " - "node[name: %s, type: %s] to %s failed.", index, mbatch_node->GetName().c_str(), + GELOGE(FAILED, "[Update][SubgraphDataOfCase] node[parent_node_index:%d] in subgraphs of " + "node[name:%s, type:%s] to %s failed.", index, mbatch_node->GetName().c_str(), mbatch_node->GetType().c_str(), TypeUtils::DataTypeToSerialString(dt_set).c_str()); return FAILED; } @@ -812,11 +840,13 @@ Status ProcessInputNC1HWC0DynShape(NodePtr &node_ptr, bool &is_dynamic_batch, No "E19014", {"opname", "value", "reason"}, {op_desc->GetName(), "format[" + TypeUtils::FormatToSerialString(old_format) + "]", "only support FORMAT_NC1HWC0,FORMAT_NCHW,FORMAT_NHWC"}); - GELOGE(INTERNAL_ERROR, "The format [%s] is unsupported", TypeUtils::FormatToSerialString(old_format).c_str()); + GELOGE(INTERNAL_ERROR, "[Check][Param] The format [%s] is unsupported, op:%s", + TypeUtils::FormatToSerialString(old_format).c_str(), op_desc->GetName().c_str()); return FAILED; } if (ModifyInputFormatAndShape(node_ptr) != SUCCESS) { - GELOGE(INTERNAL_ERROR, "modify format and shape failed"); + GELOGE(INTERNAL_ERROR, "[Modify][InputFormatAndShape] failed, op:%s(%s)", + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return FAILED; } if (is_dynamic_batch) { @@ -826,7 +856,8 @@ Status ProcessInputNC1HWC0DynShape(NodePtr &node_ptr, bool &is_dynamic_batch, No if (ModifyFormatAndShapeForSingleTensor(switchn_input) != SUCCESS) { REPORT_CALL_ERROR("E19999", "Modify format and shape of input:0 in op:%s(%s) failed", switchn_op_desc->GetName().c_str(), switchn_op_desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "modify format and shape failed"); + GELOGE(INTERNAL_ERROR, "[Modify][FormatAndShape] of input:0 in op:%s(%s) failed", + switchn_op_desc->GetName().c_str(), switchn_op_desc->GetType().c_str()); return FAILED; } for (uint32_t i = 0; i < switchn_node->GetAllOutDataAnchorsSize(); ++i) { @@ -836,8 +867,9 @@ Status ProcessInputNC1HWC0DynShape(NodePtr &node_ptr, bool &is_dynamic_batch, No old_shape = switchn_output->GetShape(); if (ModifyFormatAndShapeForSingleTensor(switchn_output) != SUCCESS) { REPORT_CALL_ERROR("E19999", "Modify format and shape of output:%u in op:%s(%s) failed", i, - switchn_op_desc->GetName().c_str(), switchn_op_desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "modify format and shape failed"); + switchn_op_desc->GetName().c_str(), switchn_op_desc->GetType().c_str()); + GELOGE(INTERNAL_ERROR, "[Modify][FormatAndShape] of output:%u in op:%s(%s) failed", i, + switchn_op_desc->GetName().c_str(), switchn_op_desc->GetType().c_str()); return FAILED; } } @@ -858,15 +890,15 @@ Status ProcessDataNodeDynShape(NodePtr &node_ptr) { NodePtr mbatch_node = nullptr; int32_t index = 0; if (CheckIfDynamicBatchScene(node_ptr, is_dynamic_batch, mbatch_node, index)) { - GELOGE(INTERNAL_ERROR, "CheckIfDynamicBatchScene failed"); + GELOGE(INTERNAL_ERROR, "[Call][CheckIfDynamicBatchScene] failed, op:%s", op_desc->GetName().c_str()); return FAILED; } if (ProcessInputDtDynShape(node_ptr, mbatch_node, dt_set) != SUCCESS) { - GELOGE(INTERNAL_ERROR, "ProcessInputFP16 failed"); + GELOGE(INTERNAL_ERROR, "[Process][InputDtDynShape] ProcessInputFP16 failed, op:%s", op_desc->GetName().c_str()); return FAILED; } if (is_dynamic_batch && ProcessMbatchScene(mbatch_node, dt_set, index) != SUCCESS) { - GELOGE(INTERNAL_ERROR, "ProcessMbatchScene failed"); + GELOGE(INTERNAL_ERROR, "[Process][MbatchScene] failed"); return FAILED; } @@ -876,7 +908,7 @@ Status ProcessDataNodeDynShape(NodePtr &node_ptr) { if (ret && (!set_format.empty()) && TypeUtils::SerialStringToFormat(set_format) == FORMAT_NC1HWC0) { GELOGI("The format of node [%s] should be set NC1HWC0.", node_ptr->GetName().c_str()); if (ProcessInputNC1HWC0DynShape(node_ptr, is_dynamic_batch, mbatch_node) != SUCCESS) { - GELOGE(INTERNAL_ERROR, "ProcessInputNC1HWC0 failed"); + GELOGE(INTERNAL_ERROR, "[Process][InputNC1HWC0] failed, op:%s", node_ptr->GetName().c_str()); return FAILED; } } @@ -905,8 +937,8 @@ Status GetStorageFormatAndShape(OpDescPtr &op_desc, const GeTensorDescPtr &tenso ErrorManager::GetInstance().ATCReportErrMessage( "15003", {"opname", "format"}, {op_desc->GetName(), TypeUtils::FormatToSerialString(storage_format)}); - GELOGE(PARAM_INVALID, "Update node by storage format failed, storage_shape not set. " - "node: [%s], storage_format [%s]", + GELOGE(PARAM_INVALID, "[Check][Param] Update node by storage format failed, storage_shape not set. " + "node:[%s], storage_format [%s]", op_desc->GetName().c_str(), TypeUtils::FormatToSerialString(storage_format).c_str()); return FAILED; } @@ -946,7 +978,9 @@ Status ProcessNetoutputNodeFp16Nc1hwc0DynShape(GeTensorDesc &src_desc, GeTensorD REPORT_CALL_ERROR("E19999", "Transfer output:0 shape of op:%s(%s) to NC1HWC0 format failed, shape:%s, format:%s", src_op_desc->GetName().c_str(), src_op_desc->GetType().c_str(), src_shape.ToString().c_str(), TypeUtils::FormatToSerialString(src_format).c_str()); - GELOGE(INTERNAL_ERROR, "Trans shape failed"); + GELOGE(INTERNAL_ERROR, "[Trans][Shape] of op:%s(%s) to NC1HWC0 format failed, shape:%s, format:%s", + src_op_desc->GetName().c_str(), src_op_desc->GetType().c_str(), + src_shape.ToString().c_str(), TypeUtils::FormatToSerialString(src_format).c_str()); return FAILED; } ge::GeShape dst_shape(dst_shape_dims); @@ -958,7 +992,8 @@ Status ProcessNetoutputNodeFp16Nc1hwc0DynShape(GeTensorDesc &src_desc, GeTensorD if (ModifyFormatAndShapeForSingleTensor(merge_out) != SUCCESS) { REPORT_CALL_ERROR("E19999", "Modify format and shape of output:0 in op:%s(%s) failed", src_op_desc->GetName().c_str(), src_op_desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "modify format and shape failed"); + GELOGE(INTERNAL_ERROR, "[Modify][FormatAndShape] of output:0 in op:%s(%s) failed", + src_op_desc->GetName().c_str(), src_op_desc->GetType().c_str()); return FAILED; } for (uint32_t i = 0; i < node->GetAllInDataAnchorsSize(); ++i) { @@ -967,7 +1002,8 @@ Status ProcessNetoutputNodeFp16Nc1hwc0DynShape(GeTensorDesc &src_desc, GeTensorD if (ModifyFormatAndShapeForSingleTensor(merge_in) != SUCCESS) { REPORT_CALL_ERROR("E19999", "Modify format and shape of input:%u in op:%s(%s) failed", i, src_op_desc->GetName().c_str(), src_op_desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "modify format and shape failed"); + GELOGE(INTERNAL_ERROR, "[Modify][FormatAndShape] of input:%u in op:%s(%s) failed", i, + src_op_desc->GetName().c_str(), src_op_desc->GetType().c_str()); return FAILED; } } @@ -1054,13 +1090,13 @@ Status ProcessNetoutputNodeDynShape(NodePtr &node) { "E19014", {"opname", "value", "reason"}, {op_desc->GetName(), "format[" + TypeUtils::FormatToSerialString(old_format) + "]", "only support FORMAT_NC1HWC0,FORMAT_NCHW,FORMAT_NHWC"}); - GELOGE(INTERNAL_ERROR, "Format is not one of NCHW, NHWC, NC1HWC0."); + GELOGE(INTERNAL_ERROR, "[Check][Param] Format is not one of NCHW, NHWC, NC1HWC0."); return FAILED; } GeTensorDesc old_desc(old_shape, old_format, old_dtype); if (ProcessNetoutputNodeFp16Nc1hwc0DynShape(old_desc, net_output_input_desc, src_node) != SUCCESS) { - GELOGE(INTERNAL_ERROR, "Process netoutput fp16 nc1hwc0."); + GELOGE(INTERNAL_ERROR, "[Process][NetOutput] fp16 nc1hwc0 failed."); return FAILED; } } @@ -1091,7 +1127,7 @@ Status GetDynamicInputShapeRange(const std::vector &user_input, const } else { REPORT_INNER_ERROR("E19999", "Graph option: %s and %s should be enabled at the same time, check invalid", OPTION_EXEC_DYNAMIC_EXECUTE_MODE, OPTION_EXEC_DATA_INPUTS_SHAPE_RANGE); - GELOGE(PARAM_INVALID, "Graph option: %s and %s should be enabled at the same time.", + GELOGE(PARAM_INVALID, "[Check][Param] Graph option: %s and %s should be enabled at the same time.", OPTION_EXEC_DYNAMIC_EXECUTE_MODE, OPTION_EXEC_DATA_INPUTS_SHAPE_RANGE); return PARAM_INVALID; } @@ -1100,8 +1136,8 @@ Status GetDynamicInputShapeRange(const std::vector &user_input, const return PARAM_INVALID; } if (range_vec.size() != user_input.size()) { - GELOGE(PARAM_INVALID, "Dynamic input shape range size is %zu, inputs size is %zu. Not match.", range_vec.size(), - user_input.size()); + GELOGE(PARAM_INVALID, "[Check][Param] Dynamic input shape range size is %zu, inputs size is %zu. Not match.", + range_vec.size(), user_input.size()); return PARAM_INVALID; } return SUCCESS; @@ -1116,8 +1152,8 @@ Status UpdateDynamicInputShapeRange(const ge::GeAttrValue::INT index, REPORT_INNER_ERROR("E19999", "Given shape_range dim num is %zu, current dim:%s num is %zu, not match, " "check invalid", current_shape_range_vec.size(), origin_shape.ToString().c_str(), origin_shape.GetDimNum()); - GELOGE(PARAM_INVALID, "Given shape_range dim num is %zu, current dim num is %zu, not match.Pleace Check.", - current_shape_range_vec.size(), origin_shape.GetDimNum()); + GELOGE(PARAM_INVALID, "[Check][Param] Given shape_range dim num is %zu, current dim num is %zu, " + "not match.Pleace Check.", current_shape_range_vec.size(), origin_shape.GetDimNum()); return PARAM_INVALID; } for (size_t i = 0; i < origin_shape.GetDimNum(); ++i) { @@ -1129,8 +1165,8 @@ Status UpdateDynamicInputShapeRange(const ge::GeAttrValue::INT index, if (curr_dim != left_range) { REPORT_INNER_ERROR("E19999", "Given shape range is %ld, current dim shape is %ld, not match, dim_index:%zu, " "check invalid", left_range, curr_dim, i); - GELOGE(PARAM_INVALID, "Given shape range is %ld, current dim shape is %ld, not match.Pleace Check.", - left_range, curr_dim); + GELOGE(PARAM_INVALID, "[Check][Param] Given shape range is %ld, current dim shape is %ld, " + "not match.Pleace Check.", left_range, curr_dim); return PARAM_INVALID; } origin_shape.SetDim(i, left_range); @@ -1141,8 +1177,8 @@ Status UpdateDynamicInputShapeRange(const ge::GeAttrValue::INT index, REPORT_INNER_ERROR("E19999", "Given shape range is [%ld~%ld], current dim shape is %ld, out of range, " "dim_index:%zu, check invalid", left_range, right_range, curr_dim, i); - GELOGE(PARAM_INVALID, "Given shape range is [%ld~%ld], current dim shape is %ld, out of range.Pleace Check.", - left_range, right_range, curr_dim); + GELOGE(PARAM_INVALID, "[Check][Param] Given shape range is [%ld~%ld], current dim shape is %ld, " + "out of range.Pleace Check.", left_range, right_range, curr_dim); return PARAM_INVALID; } } @@ -1153,9 +1189,9 @@ Status UpdateDynamicInputShapeRange(const ge::GeAttrValue::INT index, desc.SetShapeRange(current_shape_range_vec); graphStatus graph_ret = op->UpdateInputDesc(0, desc); - GE_CHK_GRAPH_STATUS_RET(graph_ret, "UpdateInputDesc fail, graph ret: %u", graph_ret); + GE_CHK_GRAPH_STATUS_RET(graph_ret, "[Update][InputDesc] fail, graph ret: %u", graph_ret); graph_ret = op->UpdateOutputDesc(0, desc); - GE_CHK_GRAPH_STATUS_RET(graph_ret, "UpdateInputDesc fail, graph ret: %u", graph_ret); + GE_CHK_GRAPH_STATUS_RET(graph_ret, "[Update][OutputDesc] fail, graph ret: %u", graph_ret); return SUCCESS; } } // namespace @@ -1188,14 +1224,14 @@ Status GraphPrepare::UpdateVariableFormats(ComputeGraphPtr &graph) { auto ret = RecoverTransRoadForVar(node, *trans_road); if (ret != SUCCESS) { - GELOGE(INTERNAL_ERROR, "Failed to recovery trans road for var %s", node->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Recover][TransRoad] for var %s failed", node->GetName().c_str()); return INTERNAL_ERROR; } auto iter = var_names_to_refs.find(node->GetName()); if (iter != var_names_to_refs.end()) { ret = RecoverTransRoadForVarRef(iter->second, *trans_road); if (ret != SUCCESS) { - GELOGE(INTERNAL_ERROR, "Failed to recovery trans road for var ref %s", node->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Recover][TransRoad] for var ref %s failed", node->GetName().c_str()); return INTERNAL_ERROR; } } @@ -1215,13 +1251,13 @@ Status GraphPrepare::Init(const ge::Graph &graph, uint64_t session_id) { Status ret = CheckGraph(); if (ret != SUCCESS) { - GELOGE(ret, "RunGraph graph check fail, ret:%u", ret); + GELOGE(ret, "[Check][Graph] fail, ret:%u", ret); return ret; } (void)compute_graph_->TopologicalSorting(); ret = CheckRefOp(); if (ret != SUCCESS) { - GELOGE(ret, "RunGraph check ref op fail, ret:%u", ret); + GELOGE(ret, "[Check][RefOp] fail, ret:%u", ret); return ret; } return SUCCESS; @@ -1230,20 +1266,20 @@ Status GraphPrepare::Init(const ge::Graph &graph, uint64_t session_id) { Status GraphPrepare::CheckGraph() { if (compute_graph_ == nullptr) { REPORT_INNER_ERROR("E19999", "compute_graph_ is nullptr, check invalid"); - GELOGE(GE_GRAPH_INIT_FAILED, "Graph prepare init compute graph is NULLPTR"); + GELOGE(GE_GRAPH_INIT_FAILED, "[Check][Param] compute_graph_ is nullptr"); return GE_GRAPH_INIT_FAILED; } auto nodes = compute_graph_->GetAllNodes(); if (nodes.empty()) { REPORT_INNER_ERROR("E19999", "nodes in graph is empty, check invalid"); - GELOGE(GE_GRAPH_INIT_FAILED, "Invalid graph, no nodes in this graph."); + GELOGE(GE_GRAPH_INIT_FAILED, "[Check][Param] Invalid graph, no nodes in this graph."); return GE_GRAPH_INIT_FAILED; } for (const NodePtr &node : compute_graph_->GetAllNodes()) { GE_CHECK_NOTNULL(node); if (node->GetOpDesc() == nullptr) { REPORT_INNER_ERROR("E19999", "node without opdesc exist in graph, check invalid"); - GELOGE(GE_GRAPH_INIT_FAILED, "Check Graph node opdesc is NULL"); + GELOGE(GE_GRAPH_INIT_FAILED, "[Get][OpDesc] failed, Check Graph node opdesc is NULL"); return GE_GRAPH_INIT_FAILED; } } @@ -1282,7 +1318,8 @@ Status GraphPrepare::CheckRefInputNode(const NodePtr &node, const std::string &i REPORT_INNER_ERROR("E19999", "Get Attr:%s of op:%s(%s) failed", ATTR_NAME_FRAMEWORK_ORIGINAL_TYPE.c_str(), input_op_desc->GetName().c_str(), input_op_desc->GetType().c_str()); - GELOGE(PARAM_INVALID, "Get original type failed."); + GELOGE(PARAM_INVALID, "[Get][Attr] %s of op:%s(%s) failed", ATTR_NAME_FRAMEWORK_ORIGINAL_TYPE.c_str(), + input_op_desc->GetName().c_str(), input_op_desc->GetType().c_str()); return PARAM_INVALID; } } @@ -1291,8 +1328,8 @@ Status GraphPrepare::CheckRefInputNode(const NodePtr &node, const std::string &i ErrorManager::GetInstance().ATCReportErrMessage( "E15005", {"opname", "optype", "opname1", "optype1"}, {op_desc->GetName(), node->GetType(), input_op_desc->GetName(), input_op_desc->GetType()}); - GELOGE(PARAM_INVALID, "The ref input of ref node %s[%s] must be ref node or variable, but %s[%s]isn't.", - node->GetName().c_str(), node->GetType().c_str(), input_op_desc->GetName().c_str(), + GELOGE(PARAM_INVALID, "[Check][Param] The ref input of ref node %s[%s] must be ref node or variable, " + "but %s[%s]isn't.", node->GetName().c_str(), node->GetType().c_str(), input_op_desc->GetName().c_str(), input_op_desc->GetType().c_str()); return PARAM_INVALID; } @@ -1306,13 +1343,13 @@ Status GraphPrepare::CheckRefOp() { for (const NodePtr &node : compute_graph_->GetDirectNode()) { if (node == nullptr) { REPORT_INNER_ERROR("E19999", "nullptr node exist in graph, check invalid"); - GELOGE(PARAM_INVALID, "param [node] must not be null."); + GELOGE(PARAM_INVALID, "[Check][Param] param [node] must not be null."); return PARAM_INVALID; } auto op_desc = node->GetOpDesc(); if (op_desc == nullptr) { REPORT_INNER_ERROR("E19999", "node without opdesc exist in graph, check invalid"); - GELOGE(PARAM_INVALID, "OpDesc of param [node] must not be null."); + GELOGE(PARAM_INVALID, "[Check][Param] OpDesc of param [node] must not be null."); return PARAM_INVALID; } @@ -1321,7 +1358,7 @@ Status GraphPrepare::CheckRefOp() { for (const auto &name_index : input_name_index) { if (op_desc->GetOutputIndexByName(name_index.first) != -1) { if (CheckRefInputNode(node, name_index.first, ref_nodes) != SUCCESS) { - GELOGE(PARAM_INVALID, "CheckRefInputNode failed."); + GELOGE(PARAM_INVALID, "[Check][RefInputNode] failed, node:%s.", op_desc->GetName().c_str()); return PARAM_INVALID; } (void)ref_nodes.insert(node); // no need to check value @@ -1346,13 +1383,13 @@ Status GraphPrepare::SetRtContext(rtContext_t rt_context, rtCtxMode_t mode) { Status GraphPrepare::AdjustDataOpOutput(const NodePtr &node) { if (node == nullptr) { REPORT_INNER_ERROR("E19999", "Param node is nullptr, check invalid"); - GELOGE(GE_GRAPH_GRAPH_NODE_NULL, "Input node is NULL"); + GELOGE(GE_GRAPH_GRAPH_NODE_NULL, "[Check][Param] Input node is nullptr"); return GE_GRAPH_GRAPH_NODE_NULL; } OpDescPtr op_desc_ptr = node->GetOpDesc(); if (op_desc_ptr == nullptr) { REPORT_INNER_ERROR("E19999", "Param node's op_desc is nullptr, check invalid"); - GELOGE(GE_GRAPH_GRAPH_NODE_NULL, "Input node opdesc is NULL"); + GELOGE(GE_GRAPH_GRAPH_NODE_NULL, "[Get][OpDesc] Input node opdesc is NULL"); return GE_GRAPH_GRAPH_NODE_NULL; } GeTensorDesc output = op_desc_ptr->GetOutputDesc(0); @@ -1367,7 +1404,7 @@ Status GraphPrepare::AdjustDataOpOutput(const NodePtr &node) { if (graph_status != GRAPH_SUCCESS) { ErrorManager::GetInstance().ATCReportErrMessage( "E19012", {"function", "reason"}, {"GetTensorMemorySizeInBytes", "opname is " + node->GetName()}); - GELOGE(graph_status, "GetTensorMemorySizeInBytes failed!"); + GELOGE(graph_status, "[Call][GetTensorMemorySizeInBytes] failed, op:%s", node->GetName().c_str()); return FAILED; } TensorUtils::SetSize(output, tensor_size); @@ -1375,7 +1412,8 @@ Status GraphPrepare::AdjustDataOpOutput(const NodePtr &node) { if (graph_ret != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Update output desc of op:%s(%s) failed, index:0", op_desc_ptr->GetName().c_str(), op_desc_ptr->GetType().c_str()); - GELOGE(graph_ret, "UpdateOutputDesc fail, graph_ret:%u", graph_ret); + GELOGE(graph_ret, "[Update][OutputDesc] of op:%s(%s) failed, index:0", + op_desc_ptr->GetName().c_str(), op_desc_ptr->GetType().c_str()); return graph_ret; } return SUCCESS; @@ -1386,7 +1424,7 @@ Status GraphPrepare::UpdateInput(const std::vector &user_input, // Get shape range of input in dynamic_execute mode vector>> dynamic_shape_range_vec; auto ret = GetDynamicInputShapeRange(user_input, graph_option, dynamic_shape_range_vec); - GE_CHK_STATUS_RET(ret, "Graph option is not right on Dynamic execute mode."); + GE_CHK_STATUS_RET(ret, "[Get][DynamicInputShapeRange] failed, Graph option is not right on Dynamic execute mode."); compute_graph_->SaveDataFormat(ge::TypeUtils::DomiFormatToFormat(GetLocalOmgContext().format)); for (NodePtr &input_node : compute_graph_->GetDirectNode()) { GE_CHECK_NOTNULL(input_node); @@ -1403,7 +1441,8 @@ Status GraphPrepare::UpdateInput(const std::vector &user_input, std::string situation = "data op index[" + std::to_string(index) + "]"; std::string reason = "it must less than user_input size[" + std::to_string(user_input.size()) + "]"; ErrorManager::GetInstance().ATCReportErrMessage("E19025", {"situation", "reason"}, {situation, reason}); - GELOGE(PARAM_INVALID, "user_input size = %zu, graph data op index = %ld.", user_input.size(), index); + GELOGE(PARAM_INVALID, "[Check][Param] user_input size = %zu, graph data op index = %ld.", + user_input.size(), index); return FAILED; } @@ -1422,7 +1461,7 @@ Status GraphPrepare::UpdateInput(const std::vector &user_input, ErrorManager::GetInstance().ATCReportErrMessage("E19025", {"situation", "reason"}, {"Input format[" + TypeUtils::FormatToSerialString(format) + "] or origin_format[" + TypeUtils::FormatToSerialString(origin_format) + "]", "it is not support"}); - GELOGE(PARAM_INVALID, "Input format %s or origin_format %s is not support.", + GELOGE(PARAM_INVALID, "[Check][Param] Input format %s or origin_format %s is not support.", TypeUtils::FormatToSerialString(format).c_str(), TypeUtils::FormatToSerialString(origin_format).c_str()); return FAILED; @@ -1435,7 +1474,7 @@ Status GraphPrepare::UpdateInput(const std::vector &user_input, if (!type_ret) { ErrorManager::GetInstance().ATCReportErrMessage("E19025", {"situation", "reason"}, {"Input datatype[" + TypeUtils::DataTypeToSerialString(data_type) + "]", "it is not support"}); - GELOGE(PARAM_INVALID, "Input datatype %s is not support.", + GELOGE(PARAM_INVALID, "[Check][Param] Input datatype %s is not support.", TypeUtils::DataTypeToSerialString(data_type).c_str()); return FAILED; } @@ -1446,7 +1485,7 @@ Status GraphPrepare::UpdateInput(const std::vector &user_input, int64_t size = 0; GE_IF_BOOL_EXEC(ge::TensorUtils::GetSize(desc, size) != GRAPH_SUCCESS, REPORT_CALL_ERROR("E19999", "Get size of user input tensor failed, index:%ld", index); - GELOGE(INTERNAL_ERROR, "TensorUtils GetSize failed"); + GELOGE(INTERNAL_ERROR, "[Get][Size] of user input tensor failed, index:%ld", index); return FAILED); bool size_check = (size != 0 && shape_size != size); if (size_check) { @@ -1454,7 +1493,7 @@ Status GraphPrepare::UpdateInput(const std::vector &user_input, "] and shape_size[" + std::to_string(size) + "]"; std::string reason = "because size != 0 and shape_size != size"; ErrorManager::GetInstance().ATCReportErrMessage("E19025", {"situation", "reason"}, {situation, reason}); - GELOGE(PARAM_INVALID, "input data size =%ld, shape_size =%ld.", size, shape_size); + GELOGE(PARAM_INVALID, "[Check][Param] input data size = %ld, shape_size = %ld.", size, shape_size); return FAILED; } ge::TensorUtils::SetSize(desc, shape_size); @@ -1463,7 +1502,8 @@ Status GraphPrepare::UpdateInput(const std::vector &user_input, if (graph_ret != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Update input desc of op:%s(%s) failed, index:0", op->GetName().c_str(), op->GetType().c_str()); - GELOGE(graph_ret, "UpdateInputDesc fail, graph_ret:%u", graph_ret); + GELOGE(graph_ret, "[Update][InputDesc] of op:%s(%s) failed, index:0", + op->GetName().c_str(), op->GetType().c_str()); return graph_ret; } // Size will be recalculated in the build stage @@ -1472,7 +1512,8 @@ Status GraphPrepare::UpdateInput(const std::vector &user_input, if (graph_ret != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Update output desc of op:%s(%s) failed, index:0", op->GetName().c_str(), op->GetType().c_str()); - GELOGE(graph_ret, "UpdateOutputDesc fail, graph_ret:%u", graph_ret); + GELOGE(graph_ret, "[Update][OutputDesc] of op:%s(%s) failed, index:0", + op->GetName().c_str(), op->GetType().c_str()); return graph_ret; } } else { @@ -1480,13 +1521,13 @@ Status GraphPrepare::UpdateInput(const std::vector &user_input, } if (!dynamic_shape_range_vec.empty()) { ret = UpdateDynamicInputShapeRange(index, dynamic_shape_range_vec, op, desc); - GE_CHK_STATUS_RET(ret, "Fail to update dynamic input shape range on %s.", op->GetName().c_str()); + GE_CHK_STATUS_RET(ret, "[Update][DynamicInputShapeRange] on %s failed.", op->GetName().c_str()); continue; } if (!options_.train_graph_flag) { Status ret = AdjustDataOpOutput(input_node); - GE_IF_BOOL_EXEC(ret != SUCCESS, GELOGE(ret, "AdjustDataOpOutput fail, ret:%u", ret); return ret); + GE_IF_BOOL_EXEC(ret != SUCCESS, GELOGE(ret, "[Adjust][DataOpOutput] fail, ret:%u", ret); return ret); } } } @@ -1500,18 +1541,18 @@ Status GraphPrepare::TryDoAipp() { GE_DUMP(compute_graph_, "Before_insert_aipp"); Status ret = ge::InsertNewOpUtil::Instance().Init(); if (ret != SUCCESS) { - GELOGE(INTERNAL_ERROR, "TryDoAipp: InsertNewOpUtil instance failed."); + GELOGE(INTERNAL_ERROR, "[Init][InsertNewOpUtil] failed."); return INTERNAL_ERROR; } ret = ge::InsertNewOpUtil::Instance().Parse(options_.insert_op_file.c_str()); if (ret != SUCCESS) { - GELOGE(GE_GRAPH_OPTIMIZE_INSERT_OP_PARSE_FAILED, "TryDoAipp: parse config file %s failed", + GELOGE(GE_GRAPH_OPTIMIZE_INSERT_OP_PARSE_FAILED, "[Parse][ConfigFile] %s failed", options_.insert_op_file.c_str()); return GE_GRAPH_OPTIMIZE_INSERT_OP_PARSE_FAILED; } ret = ge::InsertNewOpUtil::Instance().InsertAippOps(compute_graph_, options_.insert_op_file); if (ret != SUCCESS) { - GELOGE(GE_GRAPH_OPTIMIZE_INSERT_DYN_OP_FAILED, "TryDoAipp: insert aipp op ret failed, ret:%u", ret); + GELOGE(GE_GRAPH_OPTIMIZE_INSERT_DYN_OP_FAILED, "[Insert][AippOps] failed, ret:%u", ret); return GE_GRAPH_OPTIMIZE_INSERT_DYN_OP_FAILED; } } @@ -1530,7 +1571,7 @@ Status GraphPrepare::FormatAndShapeProcess() { GE_TIMESTAMP_END(InferOriginFormat1, "GraphPrepare::InferOriginFormat1"); GE_DUMP(compute_graph_, "after_first_inferformat"); if (ret != SUCCESS) { - GELOGE(ret, "Prepare Graph first inferformat failed"); + GELOGE(ret, "[Call][InferOriginFormat] Prepare Graph first inferformat failed"); return ret; } @@ -1539,7 +1580,7 @@ Status GraphPrepare::FormatAndShapeProcess() { GE_TIMESTAMP_END(InferShapeForPreprocess, "GraphPrepare::InferShapeForPreprocess"); GE_DUMP(compute_graph_, "after_infershape"); if (ret != SUCCESS) { - GELOGE(GE_GRAPH_INFERSHAPE_FAILED, "Prepare Graph infershape failed"); + GELOGE(GE_GRAPH_INFERSHAPE_FAILED, "[Call][InferShapeForPreprocess] Prepare Graph infershape failed"); return GE_GRAPH_INFERSHAPE_FAILED; } @@ -1547,7 +1588,7 @@ Status GraphPrepare::FormatAndShapeProcess() { ret = compute_graph_->InferOriginFormat(); GE_TIMESTAMP_END(InferOriginFormat2, "GraphPrepare::InferOriginFormat2"); if (ret != SUCCESS) { - GELOGE(ret, "Prepare Graph inferformat failed"); + GELOGE(ret, "[Call][InferOriginFormat] Prepare Graph inferformat failed"); return ret; } @@ -1571,13 +1612,13 @@ Status GraphPrepare::ResourcePairProcess(const std::string &action) { } } catch (std::bad_alloc &e) { REPORT_INNER_ERROR("E19999", "bad memory allocation occur when add ResourcePair Pass"); - GELOGE(INTERNAL_ERROR, "Add pass failed, bad memory allocation occur, action:%s.", action.c_str()); + GELOGE(INTERNAL_ERROR, "[Add][Pass] failed, bad memory allocation occur, action:%s.", action.c_str()); return INTERNAL_ERROR; } } Status ret = control_pass.Run(compute_graph_); if (ret != SUCCESS && ret != NOT_CHANGED) { - GELOGE(ret, "Run ResourcePairControlPass failed, action:%s, ret:%u.", action.c_str(), ret); + GELOGE(ret, "[Run][ResourcePairControlPass] failed, action:%s, ret:%u.", action.c_str(), ret); return ret; } return SUCCESS; @@ -1594,7 +1635,8 @@ Status GraphPrepare::UpdateDataNetOutputByStorageFormat() { Format storage_format = FORMAT_RESERVED; vector dst_shape_dims; if (GetStorageFormatAndShape(op_desc, input, storage_format, dst_shape_dims) != SUCCESS) { - GELOGE(INTERNAL_ERROR, "Get storage format for input failed"); + GELOGE(INTERNAL_ERROR, "[Get][StorageFormatAndShape] for input failed, op:%s, index:0", + op_desc->GetName().c_str()); return FAILED; } @@ -1603,7 +1645,8 @@ Status GraphPrepare::UpdateDataNetOutputByStorageFormat() { } if (ModifyDataNetOutputFormatAndShape(op_desc, index, storage_format, dst_shape_dims) != SUCCESS) { - GELOGE(INTERNAL_ERROR, "Modify format and shape for inputfailed"); + GELOGE(INTERNAL_ERROR, "[Modify][DataNetOutputFormatAndShape] for input failed, op:%s, index:0", + op_desc->GetName().c_str()); return FAILED; } } @@ -1616,14 +1659,16 @@ Status GraphPrepare::UpdateDataNetOutputByStorageFormat() { Format storage_format = FORMAT_RESERVED; vector dst_shape_dims; if (GetStorageFormatAndShape(op_desc, output, storage_format, dst_shape_dims) != SUCCESS) { - GELOGE(INTERNAL_ERROR, "Get storage format from output failed"); + GELOGE(INTERNAL_ERROR, "[Get][StorageFormatAndShape] from output failed, op:%s, index:%u", + op_desc->GetName().c_str(), index); return FAILED; } if (storage_format == FORMAT_RESERVED) { continue; } if (ModifyDataNetOutputFormatAndShape(op_desc, index, storage_format, dst_shape_dims) != SUCCESS) { - GELOGE(INTERNAL_ERROR, "Modify format and shape for output failed"); + GELOGE(INTERNAL_ERROR, "[Modify][DataNetOutputFormatAndShape] for output failed, op:%s, index:%u", + op_desc->GetName().c_str(), index); return FAILED; } } @@ -1675,6 +1720,7 @@ Status GraphPrepare::PrepareDynShape(const GraphNodePtr &graph_node, const std:: PP_RUN_AND_DUMP("InsertAipp", TryDoAipp); PP_RUN_AND_DUMP("ProcessBeforeInfershape", ProcessBeforeInfershape); PP_RUN_AND_DUMP("InferFormatAndShape", FormatAndShapeProcess); + PP_RUN_AND_DUMP("CtrlFlowPreProcess", CtrlFlowPreProcess); PP_RUN_AND_DUMP("GetDynamicOutputShape", multibatch::GetDynamicOutputShape, compute_graph_); PP_RUN_AND_DUMP("ProcessAippStage2", InsertNewOpUtil::Instance().UpdateDataNodeByAipp, compute_graph_); PP_RUN("SaveOriginalGraphToOmModel", SaveOriginalGraphToOmModel); @@ -1683,6 +1729,17 @@ Status GraphPrepare::PrepareDynShape(const GraphNodePtr &graph_node, const std:: return SUCCESS; } +Status GraphPrepare::CtrlFlowPreProcess() { + PassManager graph_pass; + + // After InferShape Mark v1 control flow for unknown shape. + auto mark_force_unknown_pass = new (std::nothrow) MarkForceUnknownForCondPass; + GE_CHK_STATUS_RET(graph_pass.AddPass("PreRun::MarkForceUnknownForCondPass", mark_force_unknown_pass)); + + GE_CHK_STATUS_RET(graph_pass.Run(compute_graph_)); + return SUCCESS; +} + Status GraphPrepare::RecordAIPPInfo(ge::ComputeGraphPtr &compute_graph) { PP_RUN("RecordAIPPInfo", InsertNewOpUtil::Instance().RecordAIPPInfoToData, compute_graph_); return SUCCESS; @@ -1697,7 +1754,7 @@ Status GraphPrepare::PrepareRunningFormatRefiner() { auto ret = pass_manager.Run(compute_graph); GE_TIMESTAMP_END(pass_manager, "GraphPrepare::PrepareRunningFormatRefiner"); if (ret != SUCCESS && ret != NOT_CHANGED) { - GELOGE(ret, "Run passes for running format refiner failed, ret:%u.", ret); + GELOGE(ret, "[Run][Passes] for running format refiner failed, ret:%u.", ret); return ret; } PP_RUN_AND_DUMP("UpdateInputOutputByUserOptions", UpdateInputOutputByOptions); @@ -1708,7 +1765,7 @@ Status GraphPrepare::PrepareRunningFormatRefiner() { Status GraphPrepare::SwitchOpOptimize(ComputeGraphPtr &compute_graph) { if (compute_graph == nullptr) { REPORT_INNER_ERROR("E19999", "Param compute_graph is nullptr, check invalid"); - GELOGE(GE_GRAPH_NULL_INPUT, "Input Graph is NULL"); + GELOGE(GE_GRAPH_NULL_INPUT, "[Check][Param] Input Graph is NULL"); return GE_GRAPH_NULL_INPUT; } GEPass ge_passes(compute_graph); @@ -1718,13 +1775,13 @@ Status GraphPrepare::SwitchOpOptimize(ComputeGraphPtr &compute_graph) { hccl_group.emplace_back("HcclGroupPass", &hccl_group_pass); auto ret = ge_passes.Run(hccl_group); if (ret != SUCCESS) { - GELOGE(ret, "Run HcclGroupPass pass for preprocess failed, ret:%u.", ret); + GELOGE(ret, "[Run][HcclGroupPass] pass for preprocess failed, ret:%u.", ret); return ret; } ret = compute_graph->TopologicalSorting(); if (ret != SUCCESS) { REPORT_CALL_ERROR("E19999", "Topological sorting failed"); - GELOGE(ret, "Graph topological sort failed, ret:%u.", ret); + GELOGE(ret, "[Call][TopologicalSorting] Graph topological sort failed, ret:%u.", ret); return ret; } return SUCCESS; @@ -1735,13 +1792,13 @@ Status GraphPrepare::SwitchOpOptimize(ComputeGraphPtr &compute_graph) { Status GraphPrepare::GenerateInfershapeGraph(ConstGraphPtr graph) { if (graph == nullptr) { REPORT_INNER_ERROR("E19999", "Param graph is nullptr, check invalid"); - GELOGE(GE_GRAPH_NULL_INPUT, "Input Graph is NULL"); + GELOGE(GE_GRAPH_NULL_INPUT, "[Check][Param] Input Graph is NULL"); return GE_GRAPH_NULL_INPUT; } const Graph &const_graph = *graph; Status ret = Init(const_graph, 0); if (ret != SUCCESS) { - GELOGE(ret, "Init graph_prepare fail, ret:%u", ret); + GELOGE(ret, "[Init][GraphPrepare] fail, ret:%u", ret); return ret; } GE_DUMP(compute_graph_, "after_parser"); @@ -1750,7 +1807,7 @@ Status GraphPrepare::GenerateInfershapeGraph(ConstGraphPtr graph) { GE_DUMP(compute_graph_, "after_inferformat"); if (ret != SUCCESS) { REPORT_CALL_ERROR("E19999", "Infer OriginFormat failed"); - GELOGE(ret, "Prepare Graph inferformat failed"); + GELOGE(ret, "[Infer][OriginFormat] failed"); return ret; } InferShapePass infer_shape_pass; @@ -1760,7 +1817,7 @@ Status GraphPrepare::GenerateInfershapeGraph(ConstGraphPtr graph) { ret = ge_passes.Run(names_to_passes); GE_DUMP(compute_graph_, "after_infershape"); if (ret != SUCCESS) { - GELOGE(ret, "Run ge_passes infershape for preprocess failed, ret:%u.", ret); + GELOGE(ret, "[Run][GePasses] infershape for preprocess failed, ret:%u.", ret); return ret; } ShapeRefiner::ClearContextMap(); @@ -1777,7 +1834,7 @@ Status GraphPrepare::CheckConstOp() { auto op_desc = node_ptr->GetOpDesc(); if (op_desc == nullptr) { REPORT_INNER_ERROR("E19999", "op_desc is nullptr, check invalid"); - GELOGE(PARAM_INVALID, "Get op desc failed"); + GELOGE(PARAM_INVALID, "[Get][OpDesc] of node failed, op_desc is nullptr, node type:FRAMEWORKOP."); return PARAM_INVALID; } std::string original_type; @@ -1800,7 +1857,8 @@ Status GraphPrepare::VerifyConstOp(const NodePtr &node) { if (!(AttrUtils::GetTensor(op_desc, ATTR_NAME_WEIGHTS, ge_tensor_ptr))) { REPORT_INNER_ERROR("E19999", "Get Attr:%s of op:%s(%s) failed", ATTR_NAME_WEIGHTS.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(PARAM_INVALID, "Get value from const attr failed"); + GELOGE(PARAM_INVALID, "[Get][Attr] %s of op:%s(%s) failed", ATTR_NAME_WEIGHTS.c_str(), + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return PARAM_INVALID; } GE_CHECK_NOTNULL(ge_tensor_ptr); @@ -1816,7 +1874,8 @@ Status GraphPrepare::VerifyConstOp(const NodePtr &node) { if (!type_ret) { ErrorManager::GetInstance().ATCReportErrMessage("E19025", {"situation", "reason"}, {"Input datatype[" + TypeUtils::DataTypeToSerialString(data_type) + "]", "it is not support"}); - GELOGE(PARAM_INVALID, "Input datatype %s is not support.", TypeUtils::DataTypeToSerialString(data_type).c_str()); + GELOGE(PARAM_INVALID, "[Check][Param] Input datatype %s is not support.", + TypeUtils::DataTypeToSerialString(data_type).c_str()); return FAILED; } FMK_INT64_UINT32_MULCHECK(shape_size, length); @@ -1827,18 +1886,18 @@ Status GraphPrepare::VerifyConstOp(const NodePtr &node) { // shape = [], means it's a sclar tensor. GE_CHK_BOOL_EXEC(data_size / length == 1, ErrorManager::GetInstance().ATCReportErrMessage("E10043", {"reason"}, {"Const is invalid scalar tensor."}); - return PARAM_INVALID, "Const is invalid scalar tensor."); + return PARAM_INVALID, "[Check][Param] Const is invalid scalar tensor."); } else { // shape = [x, y, 0,...], means it's a vector tensor that value is []. GE_CHK_BOOL_EXEC(data_size == 0, ErrorManager::GetInstance().ATCReportErrMessage("E10043", {"reason"}, {"Const is invalid vector scalar."}); - return PARAM_INVALID, "Const is invalid vector scalar."); + return PARAM_INVALID, "[Check][Param] Const is invalid vector scalar."); } } else { GE_CHK_BOOL_EXEC(data_size == static_cast(shape_size * length) && data_size != 0, ErrorManager::GetInstance().ATCReportErrMessage( "E10043", {"reason"}, {"Const input data size is not equal with tensor desc shape"}); - return PARAM_INVALID, "Const input data size is not equal with tensor desc shape"); + return PARAM_INVALID, "[Check][Param] Const input data size is not equal with tensor desc shape"); } return SUCCESS; } @@ -1876,14 +1935,16 @@ Status GraphPrepare::CheckUserInput(const std::vector &user_input) { if (!(AttrUtils::GetInt(op, ATTR_NAME_INDEX, index))) { REPORT_INNER_ERROR("E19999", "Get Attr:%s of op:%s(%s) failed", ATTR_NAME_WEIGHTS.c_str(), op->GetName().c_str(), op->GetType().c_str()); - GELOGE(GE_GRAPH_INIT_FAILED, "Get index from attr failed"); + GELOGE(GE_GRAPH_INIT_FAILED, "[Get][Attr] %s of op:%s(%s) failed", ATTR_NAME_WEIGHTS.c_str(), + op->GetName().c_str(), op->GetType().c_str()); return GE_GRAPH_INIT_FAILED; } if ((index < 0) || (static_cast(index) >= user_input.size())) { std::string situation = "data op index[" + std::to_string(index) + "]"; std::string reason = "it must less than user_input size[" + std::to_string(user_input.size()) + "]"; ErrorManager::GetInstance().ATCReportErrMessage("E19025", {"situation", "reason"}, {situation, reason}); - GELOGE(GE_GRAPH_INIT_FAILED, "user_input size:%zu, data op index:%ld.", user_input.size(), index); + GELOGE(GE_GRAPH_INIT_FAILED, "[Check][Param] user_input size:%zu must larger than data op index:%ld.", + user_input.size(), index); return GE_GRAPH_INIT_FAILED; } if (IsDynamicDims(input_node)) { @@ -1940,7 +2001,7 @@ Status GraphPrepare::InferShapeForPreprocess() { if (rt_err == RT_ERROR_NONE) { Status result = SetRtContext(rtContext_t(), RT_CTX_NORMAL_MODE); if (result != SUCCESS) { - GELOGE(result, "Set rt context failed."); + GELOGE(result, "[Set][RtContext] failed, mode = RT_CTX_NORMAL_MODE."); return result; } names_to_passes.emplace_back("AicpuConstantFoldingPass", &aicpu_constant_folding_pass); @@ -1951,14 +2012,14 @@ Status GraphPrepare::InferShapeForPreprocess() { if (rt_err == RT_ERROR_NONE) { Status result = SetRtContext(rtContext_t(), RT_CTX_GEN_MODE); if (result != SUCCESS) { - GELOGE(result, "Set rt context failed."); + GELOGE(result, "[Set][RtContext] failed, mode = RT_CTX_GEN_MODE."); return result; } } } ShapeRefiner::ClearContextMap(); if (ret != SUCCESS) { - GELOGE(ret, "Run ge_passes infershape for preprocess failed, ret:%u.", ret); + GELOGE(ret, "[Run][GePasses] infershape for preprocess failed, ret:%u.", ret); return ret; } return SUCCESS; @@ -1970,7 +2031,7 @@ Status GraphPrepare::PrepareOptimize() { bool has_conflict = false; graph_optimize.CheckRWConflict(compute_graph_, has_conflict); if (has_conflict) { - GELOGE(GRAPH_PARAM_INVALID, "There has rw conflict.Stop optimize."); + GELOGE(GRAPH_PARAM_INVALID, "[Check][RWConflict] There has rw conflict.Stop optimize."); return FAILED; } PassManager original_graph_passes; @@ -1981,7 +2042,7 @@ Status GraphPrepare::PrepareOptimize() { (void)original_graph_passes.AddPass("PrepareOptimize::MarkAgnosticPass", new MarkAgnosticPass); } catch (std::bad_alloc &e) { REPORT_INNER_ERROR("E19999", "bad memory allocation occur when add Pass"); - GELOGE(INTERNAL_ERROR, "Add pass failed, bad memory allocation occurs."); + GELOGE(INTERNAL_ERROR, "[Add][Pass] failed, bad memory allocation occurs."); return INTERNAL_ERROR; } @@ -1989,7 +2050,7 @@ Status GraphPrepare::PrepareOptimize() { Status ret = original_graph_passes.Run(compute_graph_); GE_TIMESTAMP_END(original_graph_passes, "GraphPrepare::OriginalGraphPasses"); if (ret != SUCCESS && ret != NOT_CHANGED) { - GELOGE(ret, "Run graph passes optimize for preprocess failed, ret:%u.", ret); + GELOGE(ret, "[Run][GraphPasses] optimize for preprocess failed, ret:%u.", ret); return ret; } // New pass @@ -2034,7 +2095,7 @@ Status GraphPrepare::PrepareOptimize() { ret = ge_passes.Run(names_to_passes); GE_TIMESTAMP_END(names_to_passes, "GraphPrepare::NamesToPasses"); if (ret != SUCCESS) { - GELOGE(ret, "Run ge_passes optimize for preprocess failed, ret:%u.", ret); + GELOGE(ret, "[Run][GePasses] optimize for preprocess failed, ret:%u.", ret); return ret; } @@ -2045,7 +2106,7 @@ Status GraphPrepare::PrepareOptimize() { (void)graph_pass.AddPass("PrepareOptimize::HcclMemcpyPass", new HcclMemcpyPass); } catch (std::bad_alloc &e) { REPORT_INNER_ERROR("E19999", "bad memory allocation occur when add Pass"); - GELOGE(INTERNAL_ERROR, "Add pass failed, bad memory allocation occurs."); + GELOGE(INTERNAL_ERROR, "[Add][Pass] failed, bad memory allocation occurs."); return INTERNAL_ERROR; } @@ -2053,7 +2114,7 @@ Status GraphPrepare::PrepareOptimize() { ret = graph_pass.Run(compute_graph_); GE_TIMESTAMP_END(graph_passes, "GraphPrepare::GraphPasses"); if (ret != SUCCESS && ret != NOT_CHANGED) { - GELOGE(ret, "Run graph passes optimize for preprocess failed, ret:%u.", ret); + GELOGE(ret, "[Run][GraphPasses] optimize for preprocess failed, ret:%u.", ret); return ret; } // The constant for train is CONSTANTOP, and is CONSTANT for inference. They will be unified in future. @@ -2062,7 +2123,7 @@ Status GraphPrepare::PrepareOptimize() { ret = compute_graph_->TopologicalSorting(); if (ret != SUCCESS) { REPORT_CALL_ERROR("E19999", "Topological sorting failed"); - GELOGE(ret, "Graph topological sort failed, ret:%u.", ret); + GELOGE(ret, "[Call][TopologicalSorting] Graph topological sort failed, ret:%u.", ret); return ret; } @@ -2116,7 +2177,7 @@ Status GraphPrepare::ProcessBeforeInfershape() { auto ret = GEPass(compute_graph_).Run(names_to_passes); GE_TIMESTAMP_END(ProcessCondRemove, "GraphManager::ProcessCondRemove"); if (ret != SUCCESS) { - GELOGE(ret, "Run ge_passes optimize for OptimizeAfterMergeSubGraph failed, ret:%d.", ret); + GELOGE(ret, "[Run][GEPass] optimize for OptimizeAfterMergeSubGraph failed, ret:%d.", ret); return ret; } return SUCCESS; @@ -2139,7 +2200,7 @@ Status GraphPrepare::ProcessNetOutput() { auto ret = graph_passes_before_infershape.Run(compute_graph_); if ((ret != SUCCESS) && (ret != NOT_CHANGED)) { - GELOGE(ret, "Run graph_passes_before_infershape failed, ret:%d.", ret); + GELOGE(ret, "[Run][GraphPasses] before Infershape failed, ret:%d.", ret); return ret; } return SUCCESS; @@ -2154,26 +2215,26 @@ Status GraphPrepare::CheckAndUpdateInput(const std::vector &user_input auto ret = CheckUserInput(user_input); if (ret != SUCCESS) { - GELOGE(ret, "Check user input failed."); + GELOGE(ret, "[Check][UserInput] failed, ret:%u", ret); return ret; } ret = UpdateInput(user_input, graph_option); if (ret != SUCCESS) { - GELOGE(ret, "UpdateInput fail, ret:%u", ret); + GELOGE(ret, "[Update][Input] fail, ret:%u", ret); return ret; } if (user_input.size() != 0) { ret = CheckConstOp(); if (ret != SUCCESS) { - GELOGE(ret, "CheckConstOp fail, ret:%u", ret); + GELOGE(ret, "[Check][ConstOp] fail, ret:%u", ret); return ret; } } else { ret = compute_graph_->TopologicalSorting(); if (ret != SUCCESS) { REPORT_CALL_ERROR("E19999", "Topological sorting failed"); - GELOGE(ret, "graph prepare error: compute_graph_->Topological Sorting"); + GELOGE(ret, "[Call][TopologicalSorting] failed."); return FAILED; } } @@ -2182,7 +2243,7 @@ Status GraphPrepare::CheckAndUpdateInput(const std::vector &user_input Status GraphPrepare::UpdateInputOutputByOptions() { auto ret = UpdateDataNetOutputByStorageFormat(); if (ret != SUCCESS) { - GELOGE(ret, "Update format acoording to storage format failed."); + GELOGE(ret, "[Update][DataNetOutputByStorageFormat] failed."); return ret; } @@ -2193,20 +2254,20 @@ Status GraphPrepare::UpdateInputOutputByOptions() { for (auto &node_ptr : compute_graph_->GetDirectNode()) { GE_CHECK_NOTNULL(node_ptr); if (CheckIfNeedSetNdFormat(node_ptr) != SUCCESS) { - GELOGE(INTERNAL_ERROR, "Set node [%s] format ND failed", node_ptr->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Set][NdFormat] for node:%s failed", node_ptr->GetName().c_str()); return FAILED; } if (node_ptr->GetType() == DATA) { if (ProcessDataNodeDynShape(node_ptr) != SUCCESS) { - GELOGE(INTERNAL_ERROR, "Process data node failed"); + GELOGE(INTERNAL_ERROR, "[Call][ProcessDataNodeDynShape] for node:%s failed", node_ptr->GetName().c_str()); return FAILED; } } if (node_ptr->GetType() == ge::NETOUTPUT) { if (ProcessNetoutputNodeDynShape(node_ptr) != SUCCESS) { - GELOGE(INTERNAL_ERROR, "Process netoutput node failed"); + GELOGE(INTERNAL_ERROR, "[Call][ProcessNetoutputNodeDynShape] for node:%s failed", node_ptr->GetName().c_str()); return FAILED; } } diff --git a/ge/graph/preprocess/graph_preprocess.h b/ge/graph/preprocess/graph_preprocess.h index 9dc3e679..3eb5e03a 100755 --- a/ge/graph/preprocess/graph_preprocess.h +++ b/ge/graph/preprocess/graph_preprocess.h @@ -79,6 +79,7 @@ class GraphPrepare { Status ProcessNetOutput(); Status ProcessBeforeInfershape(); Status UpdateInputOutputByOptions(); + Status CtrlFlowPreProcess(); bool IsTansDataOpData(const ge::NodePtr &var_node); diff --git a/ge/graph/preprocess/insert_op/ge_aipp_op.cc b/ge/graph/preprocess/insert_op/ge_aipp_op.cc index d46cb0f3..1086b842 100755 --- a/ge/graph/preprocess/insert_op/ge_aipp_op.cc +++ b/ge/graph/preprocess/insert_op/ge_aipp_op.cc @@ -116,7 +116,8 @@ Status GetDataDimN(const ge::NodePtr &data_node, ge::Format format, int64_t &bat TypeUtils::FormatToSerialString(format), "only format " + TypeUtils::FormatToSerialString(FORMAT_NCHW) + " and " + TypeUtils::FormatToSerialString(FORMAT_NHWC) + " supported"})); - GELOGE(PARAM_INVALID, "Not support data format: %s", TypeUtils::FormatToSerialString(format).c_str()); + GELOGE(PARAM_INVALID, "[Check][Param] Not support data format:%s, node:%s", + TypeUtils::FormatToSerialString(format).c_str(), data_node->GetName().c_str()); return PARAM_INVALID; } } @@ -125,7 +126,8 @@ Status GetDataDimN(const ge::NodePtr &data_node, ge::Format format, int64_t &bat ErrorManager::GetInstance().ATCReportErrMessage("E10001", {"parameter", "value", "reason"}, {data_node->GetName() + " shape size", to_string(shape.size()), errormsg}); - GELOGE(PARAM_INVALID, "The shape size of this node [%s] which linked dynamic aipp must be in range[3, 4], but is %zu", + GELOGE(PARAM_INVALID, "[Check][Param] The shape size of this node [%s] " + "which linked dynamic aipp must be in range[3, 4], but is %zu", data_node->GetName().c_str(), shape.size()); return PARAM_INVALID; } @@ -153,7 +155,8 @@ Format GetAndCheckFormat() { case domi::DOMI_TENSOR_NHWC: return FORMAT_NHWC; default: - GELOGE(PARAM_INVALID, "Unexpected format found %d", static_cast(GetLocalOmgContext().format)); + GELOGE(PARAM_INVALID, "[Check][Param] Unexpected format found %d", + static_cast(GetLocalOmgContext().format)); return FORMAT_ND; } } @@ -163,6 +166,7 @@ Status AippOp::Init(domi::AippOpParams *aipp_params) { aipp_params_ = new (std::nothrow) domi::AippOpParams(); if (aipp_params_ == nullptr) { REPORT_CALL_ERROR("E19999", "New AippOpParams failed"); + GELOGE(FAILED, "[New][AippOpParams] failed"); return FAILED; } aipp_params_->CopyFrom(*aipp_params); @@ -182,10 +186,10 @@ Status AippOp::InsertAippToGraph(ComputeGraphPtr &graph, std::string &aippConfig std::vector> target_edges; if (this->ConvertRelatedInputNameToRank() != SUCCESS) { - GELOGE(FAILED, "AippOp: convert related input name to rank failed."); + GELOGE(FAILED, "[Call][ConvertRelatedInputNameToRank] failed."); return FAILED; } - GE_CHK_STATUS_RET(this->GetTargetPosition(graph, target_input, target_edges), "Get data nodes position failed"); + GE_CHK_STATUS_RET(this->GetTargetPosition(graph, target_input, target_edges), "[Get][TargetPosition] failed"); std::map out_anchors_to_aipp; for (auto &out_in_anchors : target_edges) { @@ -203,7 +207,12 @@ Status AippOp::InsertAippToGraph(ComputeGraphPtr &graph, std::string &aippConfig out_in_anchors.first->GetOwnerNode()->GetType().c_str(), out_in_anchors.second->GetOwnerNode()->GetName().c_str(), out_in_anchors.second->GetOwnerNode()->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to link edges for aipp node %s", aipp->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Insert][Node] %s(%s) between op:%s(%s) and op:%s:%s failed", + aipp->GetName().c_str(), aipp->GetType().c_str(), + out_in_anchors.first->GetOwnerNode()->GetName().c_str(), + out_in_anchors.first->GetOwnerNode()->GetType().c_str(), + out_in_anchors.second->GetOwnerNode()->GetName().c_str(), + out_in_anchors.second->GetOwnerNode()->GetType().c_str()); return INTERNAL_ERROR; } @@ -211,7 +220,7 @@ Status AippOp::InsertAippToGraph(ComputeGraphPtr &graph, std::string &aippConfig if (GetAippMode() == domi::AippOpParams::dynamic) { ret = CreateAippData(aipp); if (ret != SUCCESS) { - GELOGE(INTERNAL_ERROR, "Failed to create aipp data for aipp %s data %s", aipp->GetName().c_str(), + GELOGE(INTERNAL_ERROR, "[Create][AippData] for aipp %s data %s failed", aipp->GetName().c_str(), out_in_anchors.first->GetOwnerNode()->GetName().c_str()); return INTERNAL_ERROR; } @@ -226,7 +235,7 @@ Status AippOp::InsertAippToGraph(ComputeGraphPtr &graph, std::string &aippConfig aipp->GetName().c_str(), aipp->GetType().c_str(), out_in_anchors.second->GetOwnerNode()->GetName().c_str(), out_in_anchors.second->GetOwnerNode()->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to link aipp %s to the peer node %s", aipp->GetName().c_str(), + GELOGE(INTERNAL_ERROR, "[Call][LinkFrom] Failed to link aipp %s to the peer node %s", aipp->GetName().c_str(), out_in_anchors.second->GetOwnerNode()->GetName().c_str()); return INTERNAL_ERROR; } @@ -242,7 +251,7 @@ NodePtr AippOp::CreateAipp(const OutDataAnchorPtr &out_anchor, auto aipp_opdesc_ptr = MakeShared(current_name, AIPP); if (aipp_opdesc_ptr == nullptr) { REPORT_CALL_ERROR("E19999", "New OpDesc failed"); - GELOGE(OUT_OF_MEMORY, "Failed to alloc aipp desc, name %s", current_name.c_str()); + GELOGE(OUT_OF_MEMORY, "[New][OpDesc] failed, name %s", current_name.c_str()); return nullptr; } @@ -271,8 +280,8 @@ NodePtr AippOp::CreateAipp(const OutDataAnchorPtr &out_anchor, REPORT_CALL_ERROR("E19999", "Update the output desc from node:%s(%s) to aipp:%s(%s) failed", node_desc->GetName().c_str(), node_desc->GetType().c_str(), aipp_opdesc_ptr->GetName().c_str(), aipp_opdesc_ptr->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to update the output desc from node %s to aipp %s", node_desc->GetName().c_str(), - aipp_opdesc_ptr->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Call][UpdateInputDesc] Failed to update the output desc from node %s to aipp %s", + node_desc->GetName().c_str(), aipp_opdesc_ptr->GetName().c_str()); return nullptr; } @@ -283,25 +292,33 @@ Status AippOp::AddAippAttrbutes(const OpDescPtr &op_desc, const std::string &aip GeAttrValue::NAMED_ATTRS aipp_attr; ConvertParamToAttr(aipp_attr); GE_CHK_BOOL_RET_STATUS(AttrUtils::SetNamedAttrs(op_desc, ATTR_NAME_AIPP, aipp_attr), - INTERNAL_ERROR, "Set name attrs for aipp node failed"); + INTERNAL_ERROR, "[Set][NamedAttrs] %s for aipp node:%s failed", ATTR_NAME_AIPP.c_str(), + op_desc->GetName().c_str()); GE_CHK_BOOL_RET_STATUS(AttrUtils::SetStr(op_desc, kAippConfigPath, aipp_cfg_path), - INTERNAL_ERROR, "Set config file path attr for aipp node failed"); + INTERNAL_ERROR, "[Set][Attr] config file path for aipp node:%s failed", + op_desc->GetName().c_str()); std::vector empty_names; GE_CHK_BOOL_RET_STATUS(AttrUtils::SetListStr(op_desc, ATTR_NAME_DATA_DUMP_ORIGIN_OP_NAMES, empty_names), - INTERNAL_ERROR, "Set ATTR_NAME_DATA_DUMP_ORIGIN_OP_NAMES attr for aipp node failed"); + INTERNAL_ERROR, "[Set][Attr] %s for aipp node:%s failed", + ATTR_NAME_DATA_DUMP_ORIGIN_OP_NAMES.c_str(), op_desc->GetName().c_str()); GE_CHK_BOOL_RET_STATUS(AttrUtils::SetInt(op_desc, kCurrentAippIndex, index), - INTERNAL_ERROR, "Set kCurrentAippIndex attr for aipp node failed"); + INTERNAL_ERROR, "[Set][Attr] %s for aipp node:%s failed", kCurrentAippIndex, + op_desc->GetName().c_str()); // add input/output desc GeTensorDesc tensor; - GE_CHK_GRAPH_STATUS_RET(op_desc->AddInputDesc("images", tensor), "Failed to add input images for aipp node"); + GE_CHK_GRAPH_STATUS_RET(op_desc->AddInputDesc("images", tensor), + "[Add][InputDesc] images for aipp node:%s failed", op_desc->GetName().c_str()); if (GetAippMode() == domi::AippOpParams::dynamic) { - GE_CHK_GRAPH_STATUS_RET(op_desc->AddOptionalInputDesc("params", tensor), "Failed to add params for aipp node"); + GE_CHK_GRAPH_STATUS_RET(op_desc->AddOptionalInputDesc("params", tensor), + "[Call][AddOptionalInputDesc] Failed to add params for aipp node:%s", + op_desc->GetName().c_str()); } - GE_CHK_GRAPH_STATUS_RET(op_desc->AddOutputDesc("features", tensor), "Failed to add output features for aipp node"); + GE_CHK_GRAPH_STATUS_RET(op_desc->AddOutputDesc("features", tensor), + "[Add][OutputDesc] features for aipp node:%s failed", op_desc->GetName().c_str()); return SUCCESS; } @@ -335,7 +352,7 @@ Status AippOp::GetAndCheckTarget(const ComputeGraphPtr &graph, int rank, NodePtr std::set &edge_indexes) { auto data_node = FindDataByIndex(graph, rank); if (data_node == nullptr) { - GELOGE(PARAM_INVALID, "Get target input node for rank %d failed", rank); + GELOGE(PARAM_INVALID, "[Call][FindDataByIndex] Get target input node for rank %d failed", rank); return PARAM_INVALID; } data_node_linked_aipp = data_node; @@ -345,7 +362,7 @@ Status AippOp::GetAndCheckTarget(const ComputeGraphPtr &graph, int rank, NodePtr if (ge::AttrUtils::GetStr(data_opdesc, ATTR_ATC_USER_DEFINE_DATATYPE, set_dt_str)) { ErrorManager::GetInstance().ATCReportErrMessage("E10034", {"opname"}, {data_opdesc->GetName()}); GELOGE(INTERNAL_ERROR, - "This input op [%s] is linked to aipp, can not be set to fp16, " + "[Get][Attr] This input op [%s] is linked to aipp, can not be set to fp16, " "please check your atc parameter --insert_op_conf, --input_fp16_nodes.", data_opdesc->GetName().c_str()); return PARAM_INVALID; @@ -364,7 +381,8 @@ Status AippOp::GetAndCheckTarget(const ComputeGraphPtr &graph, int rank, NodePtr if (!AttrUtils::SetNamedAttrs(data_opdesc, ATTR_NAME_AIPP, aipp_attr)) { REPORT_INNER_ERROR("E19999", "Set Attr:%s for op:%s(%s) failed", ATTR_NAME_AIPP.c_str(), data_opdesc->GetName().c_str(), data_opdesc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Set name attrs for Data node failed. id: %d", rank); + GELOGE(INTERNAL_ERROR, "[Set][Attr] %s for op:%s(%s) failed", ATTR_NAME_AIPP.c_str(), + data_opdesc->GetName().c_str(), data_opdesc->GetType().c_str()); return INTERNAL_ERROR; } @@ -397,15 +415,15 @@ Status AippOp::GetStaticTargetNode(const ComputeGraphPtr &graph, NodePtr &data_n REPORT_INNER_ERROR("E19999", "The data node %s has switchn node flag, but the value of attr:%s is empty, " "check invalid", data_node->GetName().c_str(), kMbatchSwitchnName); - GELOGE(INTERNAL_ERROR, "The data node %s has switchn node flag, but the value is empty", - data_node->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Check][Param] The data node %s has switchn node flag, but the value of attr:%s is empty", + data_node->GetName().c_str(), kMbatchSwitchnName); return INTERNAL_ERROR; } auto switchn = graph->FindNode(related_node_name); if (switchn == nullptr) { REPORT_INNER_ERROR("E19999", "The data node %s has switchn node %s, but can not find it on the graph, " "check invalid", data_node->GetName().c_str(), related_node_name.c_str()); - GELOGE(INTERNAL_ERROR, "The data node %s has switchn node %s, but can not find it on the graph", + GELOGE(INTERNAL_ERROR, "[Check][Param] The data node %s has switchn node %s, but can not find it on the graph", data_node->GetName().c_str(), related_node_name.c_str()); return INTERNAL_ERROR; } @@ -474,7 +492,7 @@ Status AippOp::GetTargetPosition(ComputeGraphPtr graph, NodePtr &target_input, const uint32_t related_input_rank = aipp_params_->related_input_rank(); auto ret = GetAndCheckTarget(graph, related_input_rank, target_input, edge_indexes); if (ret != SUCCESS) { - GELOGE(ret, "Get target input node for rank %u failed", related_input_rank); + GELOGE(ret, "[Get][TargetInputNode] for rank %u failed", related_input_rank); return ret; } @@ -497,13 +515,14 @@ Status AippOp::GetTargetPosition(ComputeGraphPtr graph, NodePtr &target_input, REPORT_INNER_ERROR("E19999", "Subgraph:%s of op:%s(%s) not find in graph:%s, check invalid", name.c_str(), func_desc->GetName().c_str(), func_desc->GetType().c_str(), graph->GetName().c_str()); - GELOGE(GE_GRAPH_EMPTY_SUBGRAPH, "Subgraph not found, name: %s", name.c_str()); + GELOGE(GE_GRAPH_EMPTY_SUBGRAPH, "[Get][Subgraph] failed, Subgraph:%s of op:%s(%s) not find in graph:%s", + name.c_str(), func_desc->GetName().c_str(), func_desc->GetType().c_str(), graph->GetName().c_str()); return GE_GRAPH_EMPTY_SUBGRAPH; } auto data_node = FindDataByIndex(subgraph, related_input_rank); if (data_node == nullptr) { - GELOGE(PARAM_INVALID, "Get target input node for rank %d failed", related_input_rank); + GELOGE(PARAM_INVALID, "[Get][TargetInputNode] for rank %d failed", related_input_rank); return PARAM_INVALID; } @@ -699,14 +718,16 @@ Status AippOp::GenerateOpDesc(OpDescPtr op_desc) { (op_desc->AddInputDesc(GeTensorDesc()) != GRAPH_SUCCESS)) { REPORT_CALL_ERROR("E19999", "Add input desc into op:%s(%s) failed", op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(FAILED, "failed to add input desc"); + GELOGE(FAILED, "[Add][InputDesc] into op:%s(%s) failed", + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return FAILED; } if (op_desc->AddOutputDesc(GeTensorDesc()) != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Add output desc into op:%s(%s) failed", op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(FAILED, "add output desc failed."); + GELOGE(FAILED, "[Add][OutputDesc] into op:%s(%s) failed", + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return FAILED; } GeAttrValue::NAMED_ATTRS aipp_attrs; @@ -715,7 +736,8 @@ Status AippOp::GenerateOpDesc(OpDescPtr op_desc) { GE_IF_BOOL_EXEC(!AttrUtils::SetNamedAttrs(op_desc, ATTR_NAME_AIPP, aipp_attrs), REPORT_INNER_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed", ATTR_NAME_AIPP.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(FAILED, "failed to set ATTR_NAME_AIPP"); + GELOGE(FAILED, "[Set][Attr] %s to op:%s(%s) failed", ATTR_NAME_AIPP.c_str(), + op_desc->GetName().c_str(), op_desc->GetType().c_str()); return FAILED); return SUCCESS; @@ -791,7 +813,7 @@ Status AippOp::CreateAippData(const NodePtr &aipp_node) { auto ori_data_format = GetAndCheckFormat(); if (ori_data_format != FORMAT_NCHW && ori_data_format != FORMAT_NHWC) { string format_str = TypeUtils::FormatToSerialString(ori_data_format); - GELOGE(PARAM_INVALID, "when dynamic aipp, input_format must be NCHW or NHWC, but [%s] format is %s", + GELOGE(PARAM_INVALID, "[Check][Param] when dynamic aipp, input_format must be NCHW or NHWC, but [%s] format is %s", data_node->GetName().c_str(), format_str.c_str()); string reason = "format must be NCHW or NHWC in dynamic aipp process"; ErrorManager::GetInstance().ATCReportErrMessage("E19014", {"opname", "value", "reason"}, @@ -897,7 +919,8 @@ Status AippOp::AddNodeToGraph(const NodePtr &aipp_node, int64_t max_dynamic_aipp if (stat1 != GRAPH_SUCCESS || stat2 != GRAPH_SUCCESS || stat3 != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Add and Update InputDesc to op:%s(%s) failed, index:%d", aipp_node->GetName().c_str(), aipp_node->GetType().c_str(), kAippParamsInputIndex); - GELOGE(INTERNAL_ERROR, "node process desc failed!"); + GELOGE(INTERNAL_ERROR, "[Update][InputDesc] to op:%s(%s) failed, index:%d", + aipp_node->GetName().c_str(), aipp_node->GetType().c_str(), kAippParamsInputIndex); return INTERNAL_ERROR; } // aipp_node should have two input data but now tbe only one input @@ -907,7 +930,10 @@ Status AippOp::AddNodeToGraph(const NodePtr &aipp_node, int64_t max_dynamic_aipp aipp_data_node_ptr->GetName().c_str(), aipp_data_node_ptr->GetType().c_str(), kAippDataOutputIndex, aipp_node->GetName().c_str(), aipp_node->GetType().c_str(), kAippParamsInputIndex); - GELOGE(INTERNAL_ERROR, "Add Anchor anchor between aipp data node and aipp failed!"); + GELOGE(INTERNAL_ERROR, "[Add][Edge] between op:%s(%s)(out_index:%u) and op:%s(%s)(in_index:%u) failed", + aipp_data_node_ptr->GetName().c_str(), aipp_data_node_ptr->GetType().c_str(), + kAippDataOutputIndex, aipp_node->GetName().c_str(), aipp_node->GetType().c_str(), + kAippParamsInputIndex); return INTERNAL_ERROR; } return SUCCESS; diff --git a/ge/graph/preprocess/insert_op/util_insert_aipp_op.cc b/ge/graph/preprocess/insert_op/util_insert_aipp_op.cc index 3bc8e3e4..4164a872 100755 --- a/ge/graph/preprocess/insert_op/util_insert_aipp_op.cc +++ b/ge/graph/preprocess/insert_op/util_insert_aipp_op.cc @@ -69,23 +69,23 @@ Status InsertNewOpUtil::Parse(const char *conf_path) { return SUCCESS; } - GE_CHK_BOOL_RET_STATUS(ReadProtoFromText(conf_path, insert_op_conf_.get()), FAILED, "Read AIPP conf file error: %s", - conf_path); + GE_CHK_BOOL_RET_STATUS(ReadProtoFromText(conf_path, insert_op_conf_.get()), FAILED, + "[Read][Proto] from file:%s failed", conf_path); - GE_CHK_STATUS_RET(CheckPositionNotRepeat(), "Check insert position of op failed"); + GE_CHK_STATUS_RET(CheckPositionNotRepeat(), "[Check][InsertPosition] of op failed"); for (int i = 0; i < insert_op_conf_->aipp_op_size(); i++) { domi::AippOpParams *aipp_op_params = insert_op_conf_->mutable_aipp_op(i); std::unique_ptr aipp_op(new (std::nothrow) AippOp()); GE_CHECK_NOTNULL(aipp_op); - GE_CHK_STATUS_RET(aipp_op->Init(aipp_op_params), "Aipp op init failed."); + GE_CHK_STATUS_RET(aipp_op->Init(aipp_op_params), "[Call][Init] Aipp op init failed."); insert_ops_.push_back(std::move(aipp_op)); } for (auto &dynamic_op : insert_ops_) { GE_CHECK_NOTNULL(dynamic_op); - GE_CHK_STATUS_RET(dynamic_op->ValidateParams(), "Validate insert_op config file failed"); - GE_CHK_STATUS_RET(dynamic_op->SetDefaultParams(), "Set default value of insert_op failed"); + GE_CHK_STATUS_RET(dynamic_op->ValidateParams(), "[Call][ValidateParams] Validate insert_op config file failed"); + GE_CHK_STATUS_RET(dynamic_op->SetDefaultParams(), "[Call][SetDefaultParams] Set default value of insert_op failed"); } return SUCCESS; @@ -94,12 +94,13 @@ Status InsertNewOpUtil::Parse(const char *conf_path) { Status InsertNewOpUtil::InsertAippOps(ComputeGraphPtr &graph, std::string &aippConfigPath) { GE_CHECK_NOTNULL(graph); for (uint32_t index = 0; index < insert_ops_.size(); ++index) { - GE_CHK_STATUS_RET(insert_ops_[index]->InsertAippToGraph(graph, aippConfigPath, index), "insert op to graph failed"); + GE_CHK_STATUS_RET(insert_ops_[index]->InsertAippToGraph(graph, aippConfigPath, index), + "[Insert][Op] to graph failed"); } - GE_CHK_STATUS_RET(CheckGraph(graph), "after inserting all ops, check graph failed"); + GE_CHK_STATUS_RET(CheckGraph(graph), "[Check][Graph] failed after inserting all ops"); - GE_CHK_GRAPH_STATUS_RET(graph->TopologicalSorting(), "after insert dynamic op, sort graph failed"); + GE_CHK_GRAPH_STATUS_RET(graph->TopologicalSorting(), "[Sort][Graph] failed after insert dynamic op"); ClearNewOps(); @@ -189,7 +190,7 @@ Status InsertNewOpUtil::CheckPositionNotRepeat() { ret = CheckInputNamePositionNotRepeat(); } if (ret != SUCCESS) { - GELOGE(FAILED, "Check position not repeat failed."); + GELOGE(FAILED, "[Check][Position] not repeat failed."); return FAILED; } @@ -257,8 +258,10 @@ Status InsertNewOpUtil::GetAippParams(const std::unique_ptr const OpDescPtr tmpOpPtr = aipp_node->GetOpDesc(); GE_CHECK_NOTNULL(tmpOpPtr); GE_CHK_BOOL_RET_STATUS(AttrUtils::GetNamedAttrs(tmpOpPtr, ATTR_NAME_AIPP, aipp_attr), FAILED, - "Aipp node should contain param aipp!"); - GE_CHK_STATUS_RET(OpUtils::ConvertAippParams(aipp_attr, aippParams.get()), "get aipp params failed"); + "[Get][Attr] %s from Aipp node:%s failed", + ATTR_NAME_AIPP.c_str(), tmpOpPtr->GetName().c_str()); + GE_CHK_STATUS_RET(OpUtils::ConvertAippParams(aipp_attr, aippParams.get()), + "[Convert][AippParams] get aipp params failed"); return SUCCESS; } @@ -309,7 +312,8 @@ Status InsertNewOpUtil::FindMaxSizeNode(const ComputeGraphPtr &graph, const Node REPORT_INNER_ERROR("E19999", "Subgraph:%s of op:%s(%s) not find in graph:%s, check invalid", name.c_str(), func_desc->GetName().c_str(), func_desc->GetType().c_str(), graph->GetName().c_str()); - GELOGE(GE_GRAPH_EMPTY_SUBGRAPH, "Subgraph not found, name: %s", name.c_str()); + GELOGE(GE_GRAPH_EMPTY_SUBGRAPH, "[Get][SubGraph] failed, Subgraph:%s of op:%s(%s) not find in graph:%s", + name.c_str(), func_desc->GetName().c_str(), func_desc->GetType().c_str(), graph->GetName().c_str()); return GE_GRAPH_EMPTY_SUBGRAPH; } @@ -331,7 +335,8 @@ Status InsertNewOpUtil::FindMaxSizeNode(const ComputeGraphPtr &graph, const Node REPORT_INNER_ERROR("E19999", "Get Attr:%s of op:%s(%s) failed", ATTR_NAME_PARENT_NODE_INDEX.c_str(), src_op->GetName().c_str(), src_op->GetType().c_str()); - GELOGE(FAILED, "Parent index not found, name: %s", src_op->GetName().c_str()); + GELOGE(FAILED, "[Get][Attr] %s of op:%s(%s) failed", ATTR_NAME_PARENT_NODE_INDEX.c_str(), + src_op->GetName().c_str(), src_op->GetType().c_str()); return FAILED; } @@ -384,16 +389,16 @@ Status InsertNewOpUtil::UpdateCaseNode(const ComputeGraphPtr &graph, const NodeP if (ret != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Update OutputDesc to op:%s(%s) failed, index:0", data_opdesc->GetName().c_str(), data_opdesc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to update data %s output using case %s", data->GetName().c_str(), - case_node->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Update][OutputDesc] to op:%s(%s) failed, index:0", + data_opdesc->GetName().c_str(), data_opdesc->GetType().c_str()); return INTERNAL_ERROR; } ret = data_opdesc->UpdateInputDesc(0, *input_desc); if (ret != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Update InputDesc to op:%s(%s) failed, index:0", data_opdesc->GetName().c_str(), data_opdesc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to update data %s input using case %s", data->GetName().c_str(), - case_node->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Update][InputDesc] to op:%s(%s) failed, index:0", + data_opdesc->GetName().c_str(), data_opdesc->GetType().c_str()); return INTERNAL_ERROR; } @@ -416,14 +421,16 @@ Status InsertNewOpUtil::UpdatePrevNodeByAipp(NodePtr &node, std::set &s if (graph_ret != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Get input size of op:%s(%s), index:0, failed", aipp_op_desc->GetName().c_str(), aipp_op_desc->GetType().c_str()); - GELOGE(FAILED, "UpdateOutputDesc fail, graph_ret:%d", graph_ret); + GELOGE(FAILED, "[Get][InputSize] of op:%s(%s), index:0, failed", + aipp_op_desc->GetName().c_str(), aipp_op_desc->GetType().c_str()); return FAILED; } GELOGI("Get input size [%ld] from aipp [%s].", size, aipp_op_desc->GetName().c_str()); if (size == 0) { REPORT_CALL_ERROR("E19999", "Tensor size of op:%s(%s) is 0, input_index:0, check invalid", aipp_op_desc->GetName().c_str(), aipp_op_desc->GetType().c_str()); - GELOGE(FAILED, "Can not get size from aipp [%s]", aipp_op_desc->GetName().c_str()); + GELOGE(FAILED, "[Check][Param] Tensor size of op:%s(%s) is 0, input_index:0", + aipp_op_desc->GetName().c_str(), aipp_op_desc->GetType().c_str()); return FAILED; } (void)AttrUtils::SetInt(aipp_input, ATTR_NAME_INPUT_ORIGIN_SIZE, size); @@ -511,16 +518,16 @@ Status InsertNewOpUtil::UpdateDataBySwitchN(const NodePtr &switchn, const NodePt if (ret != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Update OutputDesc to op:%s(%s) failed, index:0", data_opdesc->GetName().c_str(), data_opdesc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to update data %s output using switchn %s", data->GetName().c_str(), - switchn->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Update][OutputDesc] to op:%s(%s) failed, index:0", + data_opdesc->GetName().c_str(), data_opdesc->GetType().c_str()); return INTERNAL_ERROR; } ret = data_opdesc->UpdateInputDesc(0, *input_desc); if (ret != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Update InputDesc to op:%s(%s) failed, index:0", data_opdesc->GetName().c_str(), data_opdesc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to update data %s input using switchn %s", data->GetName().c_str(), - switchn->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Update][InputDesc] to op:%s(%s) failed, index:0", + data_opdesc->GetName().c_str(), data_opdesc->GetType().c_str()); return INTERNAL_ERROR; } // Update attr _mbatch_origin_input_dims for data when it is linked to aipp @@ -563,8 +570,8 @@ Status InsertNewOpUtil::GetDataRelatedNode(NodePtr &node, std::map aipp_params(new (std::nothrow) domi::AippOpParams()); ge::GeAttrValue::NAMED_ATTRS aipp_attr; GE_CHK_BOOL_RET_STATUS(AttrUtils::GetNamedAttrs(data_op, ATTR_NAME_AIPP, aipp_attr), ACL_ERROR_GE_AIPP_NOT_EXIST, - "Data node do not contain param aipp!"); - GE_CHK_STATUS_RET(OpUtils::ConvertAippParams(aipp_attr, aipp_params.get()), "get aipp params failed"); + "[Get][Attr] %s from op:%s failed", ATTR_NAME_AIPP.c_str(), data_op->GetName().c_str()); + GE_CHK_STATUS_RET(OpUtils::ConvertAippParams(aipp_attr, aipp_params.get()), "[Get][AippParams] failed"); for (auto out_data_anchor : node->GetAllOutDataAnchors()) { GE_CHECK_NOTNULL(out_data_anchor); @@ -621,7 +628,8 @@ Status InsertNewOpUtil::GetAllAipps(const NodePtr &data_node, const NodePtr &nod REPORT_INNER_ERROR("E19999", "Subgraph:%s of op:%s(%s) not find in graph:%s, check invalid", name.c_str(), op->GetName().c_str(), op->GetType().c_str(), graph->GetName().c_str()); - GELOGE(GE_GRAPH_EMPTY_SUBGRAPH, "Subgraph not found, name: %s", name.c_str()); + GELOGE(GE_GRAPH_EMPTY_SUBGRAPH, "[Get][SubGraph] Subgraph:%s of op:%s(%s) not find in graph:%s", + name.c_str(), op->GetName().c_str(), op->GetType().c_str(), graph->GetName().c_str()); return GE_GRAPH_EMPTY_SUBGRAPH; } @@ -635,7 +643,8 @@ Status InsertNewOpUtil::GetAllAipps(const NodePtr &data_node, const NodePtr &nod REPORT_INNER_ERROR("E19999", "Get Attr:%s of op:%s(%s) failed", ATTR_NAME_PARENT_NODE_INDEX.c_str(), src_op->GetName().c_str(), src_op->GetType().c_str()); - GELOGE(FAILED, "Parent index not found, name: %s", src_op->GetName().c_str()); + GELOGE(FAILED, "[Get][Attr] %s of op:%s(%s) failed", + ATTR_NAME_PARENT_NODE_INDEX.c_str(), src_op->GetName().c_str(), src_op->GetType().c_str()); return FAILED; } auto data = node->GetInDataNodes().at(parent_index); @@ -774,10 +783,11 @@ Status InsertNewOpUtil::SetModelInputDims(NodePtr &data_node, NodePtr &aipp_node } GELOGD("After set N or H/W to -1, the model input dims: %s.", formats::JoinToString(model_input_dims).c_str()); if (!AttrUtils::SetListInt(data_opdesc, ATTR_NAME_INPUT_DIMS, model_input_dims)) { - REPORT_INNER_ERROR("E19999", "Set Attr:%s of op:%s(%s) failed", + REPORT_INNER_ERROR("E19999", "Set Attr:%s on op:%s(%s) failed", ATTR_NAME_INPUT_DIMS.c_str(), data_opdesc->GetName().c_str(), data_opdesc->GetType().c_str()); - GELOGE(FAILED, "SetListInt of %s failed.", ATTR_NAME_INPUT_DIMS.c_str()); + GELOGE(FAILED, "[Set][Attr] %s on op:%s(%s) failed", + ATTR_NAME_INPUT_DIMS.c_str(), data_opdesc->GetName().c_str(), data_opdesc->GetType().c_str()); return FAILED; } } diff --git a/ge/graph/preprocess/multi_batch_copy_graph.cc b/ge/graph/preprocess/multi_batch_copy_graph.cc index 22f39d26..610b0137 100644 --- a/ge/graph/preprocess/multi_batch_copy_graph.cc +++ b/ge/graph/preprocess/multi_batch_copy_graph.cc @@ -79,7 +79,7 @@ NodePtr InsertMergeNodeToGraph(const std::string &name, size_t input_num, const OpDescPtr desc = MakeShared(); if (desc == nullptr) { REPORT_CALL_ERROR("E19999", "New OpDesc failed"); - GELOGE(OUT_OF_MEMORY, "Failed to insert merge node, name %s", name.c_str()); + GELOGE(OUT_OF_MEMORY, "[New][OpDesc] failed, name %s", name.c_str()); return nullptr; } desc->SetName(name); @@ -88,34 +88,35 @@ NodePtr InsertMergeNodeToGraph(const std::string &name, size_t input_num, const for (size_t i = 0; i < input_num; ++i) { auto ret = desc->AddInputDesc("x" + std::to_string(i), tensor_desc); GE_IF_BOOL_EXEC(ret != GRAPH_SUCCESS, - REPORT_CALL_ERROR("E19999", "Add input desc to op:%s(%s) failed, input desc name:%s,", + REPORT_CALL_ERROR("E19999", "Add input desc to op:%s(%s) failed, input desc name:%s", desc->GetName().c_str(), desc->GetType().c_str(), ("x" + std::to_string(i)).c_str()); - GELOGE(INTERNAL_ERROR, "Failed to create merge node %s, failed to add input %zu, error-code %u", - name.c_str(), i, ret); + GELOGE(INTERNAL_ERROR, "[Add][InputDesc] to op:%s(%s) failed, input desc name:%s", + desc->GetName().c_str(), desc->GetType().c_str(), ("x" + std::to_string(i)).c_str()); return nullptr); } auto ret = desc->AddOutputDesc("y", tensor_desc); GE_IF_BOOL_EXEC(ret != GRAPH_SUCCESS, - REPORT_CALL_ERROR("E19999", "Add output desc to op:%s(%s) failed, output desc name:%s,", + REPORT_CALL_ERROR("E19999", "Add output desc to op:%s(%s) failed, output desc name:%s", desc->GetName().c_str(), desc->GetType().c_str(), "y"); - GELOGE(INTERNAL_ERROR, "Failed to create merge node %s, failed to add output 'y', error-code %u", - name.c_str(), ret); + GELOGE(INTERNAL_ERROR, "[Add][OutputDesc] to op:%s(%s) failed, output desc name:y", + desc->GetName().c_str(), desc->GetType().c_str()); return nullptr); tensor_desc.SetDataType(DT_INT32); ret = desc->AddOutputDesc("value_index", tensor_desc); if (ret != GRAPH_SUCCESS) { - REPORT_CALL_ERROR("E19999", "Add output desc to op:%s(%s) failed, output desc name:%s,", + REPORT_CALL_ERROR("E19999", "Add output desc to op:%s(%s) failed, output desc name:%s", desc->GetName().c_str(), desc->GetType().c_str(), "value_index"); - GELOGE(INTERNAL_ERROR, "Failed to create merge node %s, failed to add output 'value_index', error-code %u", - name.c_str(), ret); + GELOGE(INTERNAL_ERROR, "[Add][OutputDesc] to op:%s(%s) failed, output desc name:value_index", + desc->GetName().c_str(), desc->GetType().c_str()); return nullptr; } if (!AttrUtils::SetBool(desc, ATTR_INSERT_BY_MBATCH, true)) { REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed", ATTR_INSERT_BY_MBATCH.c_str(), desc->GetName().c_str(), desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to create merge node %s, failed to add attr", name.c_str()); + GELOGE(INTERNAL_ERROR, "[Set][Attr] %s to op:%s(%s) failed", ATTR_INSERT_BY_MBATCH.c_str(), + desc->GetName().c_str(), desc->GetType().c_str()); return nullptr; } return graph->AddNode(desc); @@ -126,16 +127,15 @@ NodePtr InsertCopyNode(const NodePtr &node, size_t n) { auto src_op_desc = node->GetOpDesc(); GE_IF_BOOL_EXEC(src_op_desc == nullptr, REPORT_INNER_ERROR("E19999", "Param opdesc in node is nullptr, check invalid"); - GELOGE(INTERNAL_ERROR, "Failed to copy node %s to %s, the OpDesc is null", - node->GetName().c_str(), name.c_str()); + GELOGE(INTERNAL_ERROR, "[Get][OpDesc] failed, src_op_desc is nullptr"); return nullptr); auto desc = AttrUtils::CopyOpDesc(src_op_desc); GE_IF_BOOL_EXEC(desc == nullptr, REPORT_CALL_ERROR("E19999", "Copy OpDesc from op:%s(%s) failed", src_op_desc->GetName().c_str(), src_op_desc->GetType().c_str()); - GELOGE(OUT_OF_MEMORY, "Failed to create op desc for copy node for node %s name %s", - node->GetName().c_str(), name.c_str()); + GELOGE(OUT_OF_MEMORY, "[Copy][OpDesc] from op:%s(%s) failed", + src_op_desc->GetName().c_str(), src_op_desc->GetType().c_str()); return nullptr); desc->SetName(name); @@ -156,8 +156,8 @@ NodePtr InsertCopyNode(const NodePtr &node, size_t n) { GE_IF_BOOL_EXEC(output_desc == nullptr, REPORT_INNER_ERROR("E19999", "Ouput desc of op:%s(%s) not exist, index:%u, check invalid", desc->GetName().c_str(), desc->GetType().c_str(), i); - GELOGE(INTERNAL_ERROR, "Failed to get output desc by index %u from node %s when copy from %s", i, - desc->GetName().c_str(), node->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Call][MutableOutputDesc] Ouput desc of op:%s(%s) not exist, index:%u", + desc->GetName().c_str(), desc->GetType().c_str(), i); return nullptr); output_desc->CopyAttrsFrom(src_op_desc->GetOutputDesc(i)); @@ -166,7 +166,8 @@ NodePtr InsertCopyNode(const NodePtr &node, size_t n) { if (!AttrUtils::SetStr(desc, ATTR_NAME_BATCH_LABEL, batch_label)) { REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed", ATTR_NAME_BATCH_LABEL.c_str(), desc->GetName().c_str(), desc->GetType().c_str()); - GELOGE(FAILED, "set attr ATTR_NAME_BATCH_LABEL failed, node:%s.", name.c_str()); + GELOGE(FAILED, "[Set][Attr] %s to op:%s(%s) failed", ATTR_NAME_BATCH_LABEL.c_str(), + desc->GetName().c_str(), desc->GetType().c_str()); return nullptr; } @@ -189,7 +190,7 @@ NodePtr InsertConst(const std::string &name, const ComputeGraphPtr &graph) { auto desc = MakeShared(); if (desc == nullptr) { REPORT_CALL_ERROR("E19999", "New OpDesc failed"); - GELOGE(OUT_OF_MEMORY, "Failed to create const op %s, out of memory", name.c_str()); + GELOGE(OUT_OF_MEMORY, "[Create][ConstOp] %s failed, out of memory", name.c_str()); return nullptr; } desc->SetName(name); @@ -199,19 +200,22 @@ NodePtr InsertConst(const std::string &name, const ComputeGraphPtr &graph) { if (!AttrUtils::SetTensor(desc, ATTR_NAME_WEIGHTS, tensor)) { REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed", ATTR_NAME_WEIGHTS.c_str(), desc->GetName().c_str(), desc->GetType().c_str()); - GELOGE(OUT_OF_MEMORY, "Failed to init tensor value for const %s", name.c_str()); + GELOGE(OUT_OF_MEMORY, "[Set][Attr] %s to op:%s(%s) failed", ATTR_NAME_WEIGHTS.c_str(), + desc->GetName().c_str(), desc->GetType().c_str()); return nullptr; } if (!AttrUtils::SetBool(desc, ATTR_INSERT_BY_MBATCH, true)) { REPORT_CALL_ERROR("E19999", "Set Attr:%s to op:%s(%s) failed", ATTR_INSERT_BY_MBATCH.c_str(), desc->GetName().c_str(), desc->GetType().c_str()); - GELOGE(OUT_OF_MEMORY, "Failed to set insert flag for const node %s", name.c_str()); + GELOGE(OUT_OF_MEMORY, "[Set][Attr] %s to op:%s(%s) failed", ATTR_INSERT_BY_MBATCH.c_str(), + desc->GetName().c_str(), desc->GetType().c_str()); return nullptr; } if (desc->AddOutputDesc(GeTensorDesc()) != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Add output desc to op:%s(%s) failed", desc->GetName().c_str(), desc->GetType().c_str()); - GELOGE(OUT_OF_MEMORY, "Failed to add output desc for const node %s", name.c_str()); + GELOGE(OUT_OF_MEMORY, "[Add][OutputDesc] to op:%s(%s) failed", + desc->GetName().c_str(), desc->GetType().c_str()); return nullptr; } return graph->AddNode(desc); @@ -234,7 +238,7 @@ Status MultiBatchGraphCopyer::CopyGraph() { } if (LabelStatus() != SUCCESS) { - GELOGE(INTERNAL_ERROR, "Failed to label status for all nodes."); + GELOGE(INTERNAL_ERROR, "[Label][Status] for all nodes failed."); return INTERNAL_ERROR; } @@ -257,7 +261,7 @@ Status MultiBatchGraphCopyer::CopyGraph() { PrunePass prune_pass; ret = prune_pass.Run(graph_); if (ret != SUCCESS) { - GELOGE(ret, "Failed to prune"); + GELOGE(ret, "[Run][PrunePass] failed."); return ret; } return CheckCopyResult(origin_data_nodes_); @@ -271,13 +275,13 @@ Status MultiBatchGraphCopyer::Init() { ret = RelinkConstCtrlEdge(); if (ret != SUCCESS) { - GELOGE(FAILED, "Relink const's control edge failed."); + GELOGE(FAILED, "[Relink][ConstCtrlEdge] failed."); return FAILED; } ret = ExtractUnchangedStructureOutofCycle(); if (ret != SUCCESS) { - GELOGE(FAILED, "Extract unchanged structure out of cycle failed."); + GELOGE(FAILED, "[Call][ExtractUnchangedStructureOutofCycle] failed."); return FAILED; } @@ -343,13 +347,13 @@ Status MultiBatchGraphCopyer::RelinkConstCtrlEdge() { Status MultiBatchGraphCopyer::ExtractUnchangedStructureOutofCycle() { map> frame_enter; if (GetEnterNodesGroupByFrame(frame_enter) != SUCCESS) { - GELOGE(FAILED, "Get enter nodes grouped by frame_name failed."); + GELOGE(FAILED, "[Call][GetEnterNodesGroupByFrame] failed."); return FAILED; } queue nodes_to_extract; if (GetNodeNeedExtract(frame_enter, nodes_to_extract) != SUCCESS) { - GELOGE(FAILED, "Get nodes needed to extract failed."); + GELOGE(FAILED, "[Call][GetNodeNeedExtract] failed."); return FAILED; } @@ -358,17 +362,17 @@ Status MultiBatchGraphCopyer::ExtractUnchangedStructureOutofCycle() { nodes_to_extract.pop(); OpDescPtr enter_desc = nullptr; if (MoveInEntersInDataAnchorDown(node, enter_desc) != SUCCESS) { - GELOGE(FAILED, "Move in enter nodes' in data anchors down of %s failed.", node->GetName().c_str()); + GELOGE(FAILED, "[Call][MoveInEntersInDataAnchorDown] for node:%s failed.", node->GetName().c_str()); return FAILED; } set out_nodes; if (InsertEnterAfterNode(node, enter_desc, out_nodes) != SUCCESS) { - GELOGE(FAILED, "Insert enter node after %s failed.", node->GetName().c_str()); + GELOGE(FAILED, "[Insert][EnterNode] after node:%s failed.", node->GetName().c_str()); return FAILED; } if (MoveCtrlEdgeToOutNodes(node, out_nodes) != SUCCESS) { - GELOGE(FAILED, "Move %s's control edge to out nodes failed.", node->GetName().c_str()); + GELOGE(FAILED, "[Call][MoveCtrlEdgeToOutNodes] for node:%s failed.", node->GetName().c_str()); return FAILED; } @@ -381,7 +385,7 @@ Status MultiBatchGraphCopyer::ExtractUnchangedStructureOutofCycle() { } if (DeleteEnterWithoutDataOut() != SUCCESS) { - GELOGE(FAILED, "Delete enter node without out data nodes failed."); + GELOGE(FAILED, "[Call][DeleteEnterWithoutDataOut] failed."); return FAILED; } @@ -399,10 +403,11 @@ Status MultiBatchGraphCopyer::GetEnterNodesGroupByFrame(mapGetName().c_str(), op_desc->GetType().c_str()); - GELOGE(FAILED, "Get attr frame_name of enter[%s] failed.", node->GetName().c_str()); + GELOGE(FAILED, "[Get][Attr] %s from op:%s(%s) failed", + ENTER_ATTR_FRAME_NAME.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); return FAILED; } frame_enter[frame_name].emplace_back(node); @@ -611,7 +616,7 @@ Status MultiBatchGraphCopyer::LabelInBatchBranchStatus() { auto op_desc = data->GetOpDesc(); GE_IF_BOOL_EXEC(op_desc == nullptr, REPORT_INNER_ERROR("E19999", "op_desc in origin_data_nodes_ is nullptr, check invalid"); - GELOGE(PARAM_INVALID, "Op desc is nullptr."); + GELOGE(PARAM_INVALID, "[Get][OpDesc] failed, op_desc is nullptr."); return PARAM_INVALID); LabelStatusForData(data); if (!GetLocalOmgContext().dynamic_node_type.empty()) { @@ -686,7 +691,7 @@ void MultiBatchGraphCopyer::ResetEnterStatus(map> &frame Status MultiBatchGraphCopyer::LabelStatus() { if (LabelInBatchBranchStatus() != SUCCESS) { - GELOGE(PARAM_INVALID, "Failed to label no in batch branch"); + GELOGE(PARAM_INVALID, "[Call][LabelInBatchBranchStatus] failed."); return PARAM_INVALID; } @@ -751,18 +756,18 @@ Status MultiBatchGraphCopyer::CheckAndParseDynamicData(){ if (iter == data_name_order_.end()) { if (dynamic_type_ == DynamicType::kDynamicBatch) { auto ret = CheckDynamicBatchShape(data_shape_dims, data_name); - GE_IF_BOOL_EXEC(ret == false, GELOGE(PARAM_INVALID, "Failed to check dynamic batch shape of %s.", + GE_IF_BOOL_EXEC(ret == false, GELOGE(PARAM_INVALID, "[Check][DynamicBatchShape] of %s failed.", data_name.c_str()); return PARAM_INVALID); } else if (dynamic_type_ == DynamicType::kDynamicImageSize) { auto ret = CheckDynamicImageSizeShape(data_shape_dims, data_name, data_format); - GE_IF_BOOL_EXEC(ret == false, GELOGE(PARAM_INVALID, "Failed to check dynamic image size shape of %s.", + GE_IF_BOOL_EXEC(ret == false, GELOGE(PARAM_INVALID, "[Check][DynamicImageSizeShape] of %s failed.", data_name.c_str()); return PARAM_INVALID); } else if (dynamic_type_ == DynamicType::kDynamicDims) { ErrorManager::GetInstance().ATCReportErrMessage("E10001", {"parameter", "reason"}, {"--input_shape", "all dynamic data must be set in --input_shape"}); - GELOGE(INTERNAL_ERROR, "data: %s shape:%s must be set int --input_shape", + GELOGE(INTERNAL_ERROR, "[Check][Param] data:%s shape:%s must be set int --input_shape", node->GetName().c_str(), data_shape.ToString().c_str()); return INTERNAL_ERROR; } @@ -772,11 +777,11 @@ Status MultiBatchGraphCopyer::CheckAndParseDynamicData(){ } } auto ret = ParserDataToDynamicInfo(shapes_, data_name_and_shape, data_to_dynamic_info_); - GE_CHK_STATUS_RET(ret, "Failed to parse data to dynamic info."); + GE_CHK_STATUS_RET(ret, "[Call][ParserDataToDynamicInfo] failed."); if (!getnext_sink_dynamic_dims_ && unknown_shape_count == 0) { ErrorManager::GetInstance().ATCReportErrMessage("E10040"); - GELOGE(PARAM_INVALID, - "Need unknow shape data when user set --dynamic_batch_size, --dynamic_image_size or --dynamic_dims"); + GELOGE(PARAM_INVALID, "[Check][Param] Need unknow shape data " + "when user set --dynamic_batch_size, --dynamic_image_size or --dynamic_dims"); return PARAM_INVALID; } return SUCCESS; @@ -788,8 +793,8 @@ Status MultiBatchGraphCopyer::CreateNewNodes() { } else { shape_data_ = InsertGetDynamicDimsNode(); } - GE_IF_BOOL_EXEC(shape_data_ == nullptr, GELOGE(INTERNAL_ERROR, "Failed to create the shape node for multi batch"); - return INTERNAL_ERROR); + GE_IF_BOOL_EXEC(shape_data_ == nullptr, GELOGE(INTERNAL_ERROR, "[Create][TheShapeNode] for multi batch failed"); + return INTERNAL_ERROR); GE_CHECK_NOTNULL(shape_data_->GetOpDesc()); for (const auto &node : origin_all_nodes_) { @@ -818,12 +823,12 @@ Status MultiBatchGraphCopyer::CreateNewNodes() { GELOGD("Name: %s, type: %s, status: kNodeNotSupportNode.", node->GetName().c_str(), node->GetType().c_str()); break; default: - GELOGE(INTERNAL_ERROR, "Unexpected status %d on node %s", static_cast(branch_status), + GELOGE(INTERNAL_ERROR, "[Get][NodeStatus] Unexpected status %d on node %s", static_cast(branch_status), node->GetName().c_str()); break; } if (ret != SUCCESS) { - GELOGE(ret, "Failed to deal with node %s in multi-batch process", node->GetName().c_str()); + GELOGE(ret, "[DealWith][Node] %s in multi-batch process failed", node->GetName().c_str()); return ret; } } @@ -852,8 +857,9 @@ NodePtr MultiBatchGraphCopyer::InsertMergeNode(const NodePtr &node, int index) { auto merge_node_name = node->GetName() + "_ascend_mbatch_merge_" + std::to_string(index); auto merge_node = InsertMergeNodeToGraph(merge_node_name, shapes_.size(), node->GetOwnerComputeGraph()); - GE_IF_BOOL_EXEC(merge_node == nullptr, GELOGE(INTERNAL_ERROR, "Failed to create merge node for node %s, out index %d", - node->GetName().c_str(), index); + GE_IF_BOOL_EXEC(merge_node == nullptr, + GELOGE(INTERNAL_ERROR, "[Create][MergeNode] for node %s failed, out index %d", + node->GetName().c_str(), index); return nullptr); merge_nodes[index] = merge_node; GELOGI("Create merge node %s for node %s index %d", merge_node_name.c_str(), node->GetName().c_str(), index); @@ -889,7 +895,8 @@ NodePtr MultiBatchGraphCopyer::FindSwitchnNodeForDataEdge(const OutDataAnchorPtr REPORT_INNER_ERROR("E19999", "peer op_desc of op:%s(%s)'s out_index:%d anchor exist nullptr, " "check invalid", data_node->GetName().c_str(), data_node->GetType().c_str(), output_idx); - GELOGE(INTERNAL_ERROR, "Op desc should not be nullptr."); + GELOGE(INTERNAL_ERROR, "[Get][OpDesc] failed, peer op_desc of op:%s(%s)'s out_index:%d anchor exist nullptr", + data_node->GetName().c_str(), data_node->GetType().c_str(), output_idx); return nullptr; } if (getnext_nodes_to_switchn_.at(output_idx).empty()) { @@ -903,7 +910,10 @@ NodePtr MultiBatchGraphCopyer::FindSwitchnNodeForDataEdge(const OutDataAnchorPtr "check invalid", output_idx, data_node->GetName().c_str(), data_node->GetType().c_str(), getnext_nodes_to_switchn_.size(), referenced_index, getnext_nodes_to_switchn_.at(output_idx).size()); - GELOGE(INTERNAL_ERROR, "Output idx is %d, referenced index is %zu", output_idx, referenced_index); + GELOGE(INTERNAL_ERROR, "[Check][Param] output_index:%d of op:%s(%s) >= getnext_nodes_to_switchn_.size():%zu or " + "referenced_index:%zu >= getnext_nodes_to_switchn_.at(output_idx).size():%zu", output_idx, + data_node->GetName().c_str(), data_node->GetType().c_str(), getnext_nodes_to_switchn_.size(), + referenced_index, getnext_nodes_to_switchn_.at(output_idx).size()); return nullptr; } if (peer_in_anchor->GetOwnerNode()->GetOpDesc()->GetName() == origin_node->GetName()) { @@ -936,7 +946,7 @@ Status MultiBatchGraphCopyer::CopyInDataEdges(const NodePtr &origin_node, int ba switchn->GetName().c_str(), switchn->GetType().c_str(), batch_num, copyed_node->GetName().c_str(), copyed_node->GetType().c_str(), in_anchor->GetIdx()); - GELOGE(INTERNAL_ERROR, "Failed to add data edge between %s(%d) to %s(%d), error-code %u", + GELOGE(INTERNAL_ERROR, "[Add][DataEdge] between %s(%d) and %s(%d) failed, error-code %u", switchn->GetName().c_str(), batch_num, copyed_node->GetName().c_str(), in_anchor->GetIdx(), ret); return INTERNAL_ERROR; @@ -956,7 +966,7 @@ Status MultiBatchGraphCopyer::CopyInDataEdges(const NodePtr &origin_node, int ba src_batch_node->GetType().c_str(), origin_src_anchor->GetIdx(), copyed_node->GetName().c_str(), copyed_node->GetType().c_str(), in_anchor->GetIdx()); - GELOGE(INTERNAL_ERROR, "Failed to add data edge between %s(%d) to %s(%d), error-code %u", + GELOGE(INTERNAL_ERROR, "[Add][DataEdge] between %s(%d) and %s(%d) failed, error-code %u", src_batch_node->GetName().c_str(), batch_num, copyed_node->GetName().c_str(), in_anchor->GetIdx(), ret); return INTERNAL_ERROR; } @@ -972,7 +982,7 @@ Status MultiBatchGraphCopyer::CopyInDataEdges(const NodePtr &origin_node, int ba origin_src_node->GetType().c_str(), origin_src_anchor->GetIdx(), copyed_node->GetName().c_str(), copyed_node->GetType().c_str(), in_anchor->GetIdx()); - GELOGE(INTERNAL_ERROR, "Failed to add data edge between origin node %s(%d) to copyed %s(%d)", + GELOGE(INTERNAL_ERROR, "[Add][DataEdge] between origin node %s(%d) and copyed %s(%d) failed", origin_src_node->GetName().c_str(), origin_src_anchor->GetIdx(), copyed_node->GetName().c_str(), dst_anchor->GetIdx()); return INTERNAL_ERROR; @@ -994,7 +1004,7 @@ Status MultiBatchGraphCopyer::CopyInControlEdges(const NodePtr &node, int batch_ REPORT_CALL_ERROR("E19999", "Add ctrl edge between op:%s(%s) and op:%s(%s) failed", switchn_iter->second->GetName().c_str(), switchn_iter->second->GetType().c_str(), copyed_node->GetName().c_str(), copyed_node->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to add control edge between %s to %s, error-code %u", + GELOGE(INTERNAL_ERROR, "[Add][ControlEdge] between %s and %s failed, error-code %u", switchn_iter->second->GetName().c_str(), copyed_node->GetName().c_str(), ret); return INTERNAL_ERROR; } @@ -1011,7 +1021,7 @@ Status MultiBatchGraphCopyer::CopyInControlEdges(const NodePtr &node, int batch_ REPORT_CALL_ERROR("E19999", "Add ctrl edge between op:%s(%s) and op:%s(%s) failed", src_batch_node->GetName().c_str(), src_batch_node->GetType().c_str(), copyed_node->GetName().c_str(), copyed_node->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to add data edge between %s to %s, error-code %u", + GELOGE(INTERNAL_ERROR, "[Add][ControlEdge] between %s and %s failed, error-code %u", src_batch_node->GetName().c_str(), copyed_node->GetName().c_str(), ret); return INTERNAL_ERROR; } @@ -1024,7 +1034,7 @@ Status MultiBatchGraphCopyer::CopyInControlEdges(const NodePtr &node, int batch_ REPORT_CALL_ERROR("E19999", "Add ctrl edge between op:%s(%s) and op:%s(%s) failed", origin_src_node->GetName().c_str(), origin_src_node->GetType().c_str(), copyed_node->GetName().c_str(), copyed_node->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to add control edge from origin %s to copyed %s", + GELOGE(INTERNAL_ERROR, "[Add][ControlEdge] from origin %s to copyed %s failed", origin_src_node->GetName().c_str(), copyed_node->GetName().c_str()); return INTERNAL_ERROR; } @@ -1038,7 +1048,7 @@ NodePtr MultiBatchGraphCopyer::InsertShapeDataNode() { auto desc = MakeShared(); if (desc == nullptr) { REPORT_CALL_ERROR("E19999", "New OpDesc failed"); - GELOGE(OUT_OF_MEMORY, "Failed to create shape data node, out of memory"); + GELOGE(OUT_OF_MEMORY, "[New][OpDesc] failed, out of memory"); return nullptr; } string node_name = "ascend_mbatch_shape_data"; @@ -1054,21 +1064,24 @@ NodePtr MultiBatchGraphCopyer::InsertShapeDataNode() { if (ret != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Add input desc to op:%s(%s) failed", desc->GetName().c_str(), desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to add input desc for created data"); + GELOGE(INTERNAL_ERROR, "[Add][InputDesc] to op:%s(%s) failed", + desc->GetName().c_str(), desc->GetType().c_str()); return nullptr; } ret = desc->AddOutputDesc(tensor_desc); if (ret != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Add output desc into op:%s(%s) failed", desc->GetName().c_str(), desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to add output desc for created data"); + GELOGE(INTERNAL_ERROR, "[Add][OutputDesc] into op:%s(%s) failed", + desc->GetName().c_str(), desc->GetType().c_str()); return nullptr; } if (!AttrUtils::SetBool(desc, ATTR_INSERT_BY_MBATCH, true)) { REPORT_CALL_ERROR("E19999", "Set Attr:%s to node:%s(%s) failed", ATTR_INSERT_BY_MBATCH.c_str(), desc->GetName().c_str(), desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to add attr for created data"); + GELOGE(INTERNAL_ERROR, "[Set][Attr] %s to node:%s(%s) failed", + ATTR_INSERT_BY_MBATCH.c_str(), desc->GetName().c_str(), desc->GetType().c_str()); return nullptr; } @@ -1076,7 +1089,8 @@ NodePtr MultiBatchGraphCopyer::InsertShapeDataNode() { if (data_node == nullptr) { REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s failed", desc->GetName().c_str(), desc->GetType().c_str(), graph_->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to add shape data node to graph"); + GELOGE(INTERNAL_ERROR, "[Add][Node] %s(%s) to graph:%s failed", + desc->GetName().c_str(), desc->GetType().c_str(), graph_->GetName().c_str()); return nullptr; } ret = GraphUtils::AppendInputNode(graph_, data_node); @@ -1084,7 +1098,8 @@ NodePtr MultiBatchGraphCopyer::InsertShapeDataNode() { REPORT_CALL_ERROR("E19999", "Append input node:%s(%s) to graph:%s failed", data_node->GetName().c_str(), data_node->GetType().c_str(), graph_->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to append data node %s as input to graph", data_node->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Append][InputNode] %s to graph:%s failed", + data_node->GetName().c_str(), graph_->GetName().c_str()); return nullptr; } @@ -1096,7 +1111,7 @@ NodePtr MultiBatchGraphCopyer::InsertGetDynamicDimsNode() { auto desc = MakeShared(); if (desc == nullptr) { REPORT_CALL_ERROR("E19999", "New OpDesc failed"); - GELOGE(OUT_OF_MEMORY, "Failed to create shape data node, out of memory"); + GELOGE(OUT_OF_MEMORY, "[New][OpDesc] failed, out of memory"); return nullptr; } string node_name = "ascend_mbatch_get_dynamic_dims_node"; @@ -1120,7 +1135,8 @@ NodePtr MultiBatchGraphCopyer::InsertGetDynamicDimsNode() { GE_IF_BOOL_EXEC(ret != GRAPH_SUCCESS, REPORT_CALL_ERROR("E19999", "Add input desc to op:%s(%s) failed", desc->GetName().c_str(), desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to add input desc for created data"); + GELOGE(INTERNAL_ERROR, "[Add][InputDesc] to op:%s(%s) failed", + desc->GetName().c_str(), desc->GetType().c_str()); return nullptr); continue; } @@ -1129,7 +1145,8 @@ NodePtr MultiBatchGraphCopyer::InsertGetDynamicDimsNode() { GE_IF_BOOL_EXEC(ret != GRAPH_SUCCESS, REPORT_CALL_ERROR("E19999", "Add input desc to op:%s(%s) failed", desc->GetName().c_str(), desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to add input desc for created data"); + GELOGE(INTERNAL_ERROR, "[Add][InputDesc] to op:%s(%s) failed", + desc->GetName().c_str(), desc->GetType().c_str()); return nullptr); } @@ -1138,13 +1155,15 @@ NodePtr MultiBatchGraphCopyer::InsertGetDynamicDimsNode() { GE_IF_BOOL_EXEC(ret != GRAPH_SUCCESS, REPORT_CALL_ERROR("E19999", "Add output desc to op:%s(%s) failed", desc->GetName().c_str(), desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to add output desc for created data"); + GELOGE(INTERNAL_ERROR, "[Add][OutputDesc] to op:%s(%s) failed", + desc->GetName().c_str(), desc->GetType().c_str()); return nullptr); if (!AttrUtils::SetBool(desc, ATTR_INSERT_BY_MBATCH, true)) { REPORT_CALL_ERROR("E19999", "Set Attr:%s to node:%s(%s) failed", ATTR_INSERT_BY_MBATCH.c_str(), desc->GetName().c_str(), desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to add attr for created data"); + GELOGE(INTERNAL_ERROR, "[Set][Attr] %s to node:%s(%s) failed", + ATTR_INSERT_BY_MBATCH.c_str(), desc->GetName().c_str(), desc->GetType().c_str()); return nullptr; } @@ -1152,7 +1171,8 @@ NodePtr MultiBatchGraphCopyer::InsertGetDynamicDimsNode() { if (data_node == nullptr) { REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s failed", desc->GetName().c_str(), desc->GetType().c_str(), graph_->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to add shape data node to graph"); + GELOGE(INTERNAL_ERROR, "[Add][Node] %s(%s) to graph:%s failed", + desc->GetName().c_str(), desc->GetType().c_str(), graph_->GetName().c_str()); return nullptr; } ret = GraphUtils::AppendInputNode(graph_, data_node); @@ -1160,7 +1180,8 @@ NodePtr MultiBatchGraphCopyer::InsertGetDynamicDimsNode() { REPORT_CALL_ERROR("E19999", "Append input node:%s(%s) to graph:%s failed", data_node->GetName().c_str(), data_node->GetType().c_str(), graph_->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to append data node %s as input to graph", data_node->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Append][InputNode] %s(%s) to graph:%s failed", + data_node->GetName().c_str(), data_node->GetType().c_str(), graph_->GetName().c_str()); return nullptr; } @@ -1170,7 +1191,7 @@ NodePtr MultiBatchGraphCopyer::InsertGetDynamicDimsNode() { Status MultiBatchGraphCopyer::CheckArguments() { if (graph_ == nullptr) { REPORT_INNER_ERROR("E19999", "graph_ is nullptr, check invalid"); - GELOGE(PARAM_INVALID, "Failed to copy graph, the graph is null"); + GELOGE(PARAM_INVALID, "[Check][Param] graph_ is nullptr"); return PARAM_INVALID; } @@ -1186,7 +1207,7 @@ Status MultiBatchGraphCopyer::CheckCopyResult(const std::vector &start_ if (!IsAllDimsPositive(dims)) { ErrorManager::GetInstance().ATCReportErrMessage("E15004", {"opname", "shape"}, {node->GetName(), formats::ShapeToString(dims)}); - GELOGE(INTERNAL_ERROR, "Failed to copy multi batch graph, the node %s still has unknown shape %s", + GELOGE(INTERNAL_ERROR, "[Check][Param] Failed to copy multi batch graph, the node %s still has unknown shape %s", node->GetName().c_str(), formats::ShapeToString(dims).c_str()); return INTERNAL_ERROR; } @@ -1219,7 +1240,7 @@ Status MultiBatchGraphCopyer::LinkDataToMerge(const NodePtr &data, const NodePtr REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(index:%zu) and op:%s(%s)(index:%zu) failed", switchn->GetName().c_str(), switchn->GetType().c_str(), i, merge->GetName().c_str(), merge->GetType().c_str(), i); - GELOGE(INTERNAL_ERROR, "Failed to add edge between switchn %s(%zu) to merge %s(%zu), error-code %u", + GELOGE(INTERNAL_ERROR, "[Add][Edge] between switchn %s(%zu) and merge %s(%zu) failed, ret:%u", switchn->GetName().c_str(), i, merge->GetName().c_str(), i, ret); return INTERNAL_ERROR); } @@ -1232,8 +1253,8 @@ Status MultiBatchGraphCopyer::LinkNodeToMerge(const NodePtr &node, int out_index REPORT_INNER_ERROR("E19999", "Create merge node for node %s failed, " "the copyed nodes for it count %zu different with shape %zu, check invalid", node->GetName().c_str(), copyed_nodes.size(), shapes_.size()); - GELOGE(INTERNAL_ERROR, - "Failed to create merge node for node %s, the copyed nodes for it count %zu different with shape %zu", + GELOGE(INTERNAL_ERROR, "[Check][Param] Failed to create merge node for node %s, " + "the copyed nodes for it count %zu different with shape %zu", node->GetName().c_str(), copyed_nodes.size(), shapes_.size()); return INTERNAL_ERROR; } @@ -1248,7 +1269,7 @@ Status MultiBatchGraphCopyer::LinkNodeToMerge(const NodePtr &node, int out_index src_node->GetName().c_str(), const_name.c_str()); auto const_node = InsertConst(const_name, graph_); GE_IF_BOOL_EXEC(const_node == nullptr, - GELOGE(OUT_OF_MEMORY, "Failed to create const for node %s to connect to a merge node", + GELOGE(OUT_OF_MEMORY, "[Create][Const] for node:%s failed, which to connect to a merge node", src_node->GetName().c_str()); return OUT_OF_MEMORY); @@ -1257,8 +1278,8 @@ Status MultiBatchGraphCopyer::LinkNodeToMerge(const NodePtr &node, int out_index REPORT_CALL_ERROR("E19999", "Add ctrl edge between op:%s(%s) and op:%s(%s) failed", src_node->GetName().c_str(), src_node->GetType().c_str(), const_node->GetName().c_str(), const_node->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to add control edge from %s to %s", - src_node->GetName().c_str(), const_node->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Add][ControlEdge] from %s to %s failed", + src_node->GetName().c_str(), const_node->GetName().c_str()); return INTERNAL_ERROR); src_node = const_node; @@ -1266,10 +1287,10 @@ Status MultiBatchGraphCopyer::LinkNodeToMerge(const NodePtr &node, int out_index auto ret = GraphUtils::AddEdge(src_node->GetOutDataAnchor(out_index), merge->GetInDataAnchor(i)); if (ret != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(index:%d) and op:%s(%s)(index:%zu) failed", - src_node->GetName().c_str(), src_node->GetType().c_str(), out_index, - merge->GetName().c_str(), merge->GetType().c_str(), i); + src_node->GetName().c_str(), src_node->GetType().c_str(), out_index, + merge->GetName().c_str(), merge->GetType().c_str(), i); GELOGE(INTERNAL_ERROR, - "Failed to add edge between copyed node %s(%d) to inserted merge node %s(%zu), error-code %u", + "[Add][Edge] between copyed node %s(%d) and inserted merge node %s(%zu) failed, error-code %u", copyed_nodes[i]->GetName().c_str(), out_index, merge->GetName().c_str(), i, ret); return INTERNAL_ERROR; } @@ -1281,11 +1302,11 @@ Status MultiBatchGraphCopyer::InsertSwitchNAndUpdateMaxShape(const NodePtr &node std::vector> dynamic_out_to_switchn; if (!getnext_sink_dynamic_dims_) { if (InsertSwitchNForData(node, kDataOutIndex, kDataOutIndex, dynamic_out_to_switchn) != SUCCESS) { - GELOGE(PARAM_INVALID, "Failed to insert switchn for %s.", node->GetName().c_str()); + GELOGE(PARAM_INVALID, "[Insert][SwitchN] for node:%s failed.", node->GetName().c_str()); return PARAM_INVALID; } if (UpdateMaxShapeToData(node, kDataOutIndex) != SUCCESS) { - GELOGE(PARAM_INVALID, "Failed to update max shape of %s.", node->GetName().c_str()); + GELOGE(PARAM_INVALID, "[Update][MaxShape] of node:%s failed.", node->GetName().c_str()); return PARAM_INVALID; } } else { @@ -1300,7 +1321,7 @@ Status MultiBatchGraphCopyer::InsertSwitchNAndUpdateMaxShape(const NodePtr &node getnext_sink_dynamic_out_mapping_.at(i).second); if (InsertSwitchNForData(node, getnext_sink_dynamic_out_mapping_.at(i).first, j, dynamic_out_to_switchn) != SUCCESS) { - GELOGE(PARAM_INVALID, "Failed to insert switchn for %s of %zu out anchor when referenced index is %zu", + GELOGE(PARAM_INVALID, "[Insert][SwitchN] for %s of %zu out anchor failed, when referenced index is %zu", node->GetName().c_str(), getnext_sink_dynamic_out_mapping_.at(i).first, j); return PARAM_INVALID; } @@ -1310,7 +1331,8 @@ Status MultiBatchGraphCopyer::InsertSwitchNAndUpdateMaxShape(const NodePtr &node for (size_t i = 0; i < getnext_sink_dynamic_out_mapping_.size(); ++i) { if(UpdateMaxShapeToData(node, i) != SUCCESS) { - GELOGE(PARAM_INVALID, "Failed to update %s max shape of %zu out anchor", node->GetName().c_str(), i); + GELOGE(PARAM_INVALID, "[Call][UpdateMaxShapeToData]Failed to update %s max shape of %zu out anchor", + node->GetName().c_str(), i); return PARAM_INVALID; } } @@ -1328,7 +1350,8 @@ Status MultiBatchGraphCopyer::UpdateShapeOfShapeNode(const NodePtr &node, size_t if (node->GetOpDesc()->UpdateOutputDesc(shape_index, output_desc) != SUCCESS) { REPORT_CALL_ERROR("E19999", "Update output desc to op:%s(%s) failed, index:%zu", node->GetName().c_str(), node->GetType().c_str(), shape_index); - GELOGE(FAILED, "Update output desc fail."); + GELOGE(FAILED, "[Update][OutputDesc] to op:%s(%s) failed, index:%zu", + node->GetName().c_str(), node->GetType().c_str(), shape_index); return FAILED; } return SUCCESS; @@ -1351,7 +1374,8 @@ Status MultiBatchGraphCopyer::UpdateMaxShapeToData(const NodePtr &node, size_t o } else { if (IsAllDimsPositive(data_shape.GetDims())) { // need to update shape of Shape_node - GE_CHK_STATUS_RET(UpdateShapeOfShapeNode(node, out_anchor_index), "Failed to update shape of shape node"); + GE_CHK_STATUS_RET(UpdateShapeOfShapeNode(node, out_anchor_index), + "[Update][ShapeOfShapeNode] %s failed, index:%zu", node->GetName().c_str(), out_anchor_index); return SUCCESS; } } @@ -1365,7 +1389,8 @@ Status MultiBatchGraphCopyer::UpdateMaxShapeToData(const NodePtr &node, size_t o REPORT_CALL_ERROR("E19999", "Op:%s(%s)'s shape:%s size will overflow after multi, check invalid", node->GetName().c_str(), node->GetType().c_str(), formats::ShapeToString(data_to_dynamic_info_[data_name].at(i)).c_str()); - GELOGE(PARAM_INVALID, "The shape %s size overflow", + GELOGE(PARAM_INVALID, "[Check][Param] Op:%s(%s)'s shape:%s size will overflow after multi", + node->GetName().c_str(), node->GetType().c_str(), formats::ShapeToString(data_to_dynamic_info_[data_name].at(i)).c_str()); return PARAM_INVALID; } @@ -1379,14 +1404,15 @@ Status MultiBatchGraphCopyer::UpdateMaxShapeToData(const NodePtr &node, size_t o // must not be error, the calc result has been checked in function InsertSwitchNForData (void)CalcShape(data_to_dynamic_info_.at(data_name).at(max_shape_index), data_shape); auto ret = NodeUtils::UpdateOutputShape(*node, out_anchor_index, data_shape); - GE_CHK_GRAPH_STATUS_RET(ret, "Failed to update output shape for data %s", node->GetName().c_str()); + GE_CHK_GRAPH_STATUS_RET(ret, "[Update][OutputShape] for data %s failed", node->GetName().c_str()); // getnext_sink not has input if (!getnext_sink_dynamic_dims_) { ret = NodeUtils::UpdateInputShape(*node, kDataInIndex, data_shape); - GE_CHK_GRAPH_STATUS_RET(ret, "Failed to update input shape for data %s", node->GetName().c_str()); + GE_CHK_GRAPH_STATUS_RET(ret, "[Update][InputShape] for data %s failed", node->GetName().c_str()); } else { // need to update shape of Shape_node when getnext_sink_dynamic - GE_CHK_STATUS_RET(UpdateShapeOfShapeNode(node, out_anchor_index), "Failed to update shape of shape node"); + GE_CHK_STATUS_RET(UpdateShapeOfShapeNode(node, out_anchor_index), + "[Update][ShapeOfShapeNode] %s failed, index:%zu", node->GetName().c_str(), out_anchor_index); } GELOGI("Update the data %s input/output shape to the max %s", node->GetName().c_str(), formats::ShapeToString(data_shape).c_str()); @@ -1413,7 +1439,7 @@ Status MultiBatchGraphCopyer::InsertSwitchNForData(const NodePtr &node, const si auto switchn_desc = MakeShared(); GE_IF_BOOL_EXEC(switchn_desc == nullptr, REPORT_CALL_ERROR("E19999", "New OpDesc failed"); - GELOGE(OUT_OF_MEMORY, "Failed to create switchn for data %s", node->GetName().c_str()); + GELOGE(OUT_OF_MEMORY, "[New][OpDesc] failed"); return OUT_OF_MEMORY); string switchn_name = node->GetName() + "_ascend_mbatch_switchn"; if (getnext_sink_dynamic_dims_) { @@ -1429,14 +1455,16 @@ Status MultiBatchGraphCopyer::InsertSwitchNForData(const NodePtr &node, const si REPORT_CALL_ERROR("E19999", "Add input desc to op:%s(%s) failed, input desc name:%s", switchn_desc->GetName().c_str(), switchn_desc->GetType().c_str(), "data"); - GELOGE(OUT_OF_MEMORY, "Failed to add input tensor desc for %s", switchn_desc->GetName().c_str()); + GELOGE(OUT_OF_MEMORY, "[Add][InputDesc] to op:%s(%s) failed, input desc name:data", + switchn_desc->GetName().c_str(), switchn_desc->GetType().c_str()); return OUT_OF_MEMORY); GeTensorDesc pred_tensor; GE_IF_BOOL_EXEC(switchn_desc->AddInputDesc("pred_value", pred_tensor) != GRAPH_SUCCESS, REPORT_CALL_ERROR("E19999", "Add input desc to op:%s(%s) failed, input desc name:%s", switchn_desc->GetName().c_str(), switchn_desc->GetType().c_str(), "pred_value"); - GELOGE(OUT_OF_MEMORY, "Failed to add input pred tensor desc for %s", switchn_desc->GetName().c_str()); + GELOGE(OUT_OF_MEMORY, "[Add][InputDesc] to op:%s(%s) failed, input desc name:pred_value", + switchn_desc->GetName().c_str(), switchn_desc->GetType().c_str()); return OUT_OF_MEMORY); std::vector input_dims_str; for (size_t i = 0; i < shapes_.size(); ++i) { @@ -1445,7 +1473,7 @@ Status MultiBatchGraphCopyer::InsertSwitchNForData(const NodePtr &node, const si auto shape = data_shape; auto ret = CalcShape(data_to_dynamic_info_.at(data_name).at(i), shape); if (ret != SUCCESS) { - GELOGE(ret, "Failed to calculate the batched shape for data node %s, the shapes may not match", + GELOGE(ret, "[Calc][Shape] Failed to calculate the batched shape for data node %s, the shapes may not match", node->GetName().c_str()); return ret; } @@ -1462,7 +1490,8 @@ Status MultiBatchGraphCopyer::InsertSwitchNForData(const NodePtr &node, const si REPORT_CALL_ERROR("E19999", "Set Attr:%s to output tensor of node:%s(%s) failed, index:%zu", ATTR_NAME_SWITCHN_PRED_VALUE.c_str(), node->GetName().c_str(), node->GetType().c_str(), out_anchor_index); - GELOGE(INTERNAL_ERROR, "Failed to add attr value on output %zu tensor", i); + GELOGE(INTERNAL_ERROR, "[Set][Attr] %s to output tensor of node:%s(%s) failed, index:%zu", + ATTR_NAME_SWITCHN_PRED_VALUE.c_str(), node->GetName().c_str(), node->GetType().c_str(), out_anchor_index); return INTERNAL_ERROR; } (void) AttrUtils::SetListInt(tensor, ATTR_NAME_COMBINED_DYNAMIC_DIMS, shape.GetDims()); @@ -1470,7 +1499,9 @@ Status MultiBatchGraphCopyer::InsertSwitchNForData(const NodePtr &node, const si REPORT_CALL_ERROR("E19999", "Add output desc to op:%s(%s) failed, output desc name:%s", switchn_desc->GetName().c_str(), switchn_desc->GetType().c_str(), ("output" + std::to_string(i)).c_str()); - GELOGE(GRAPH_FAILED, "Opdesc AddOutputDesc failed"); + GELOGE(GRAPH_FAILED, "[Add][OutputDesc] to op:%s(%s) failed, output desc name:%s", + switchn_desc->GetName().c_str(), switchn_desc->GetType().c_str(), + ("output" + std::to_string(i)).c_str()); return GRAPH_FAILED; } GELOGD("The switchn %s output index %zu, shape %s", switchn_desc->GetName().c_str(), i, shape.ToString().c_str()); @@ -1480,20 +1511,22 @@ Status MultiBatchGraphCopyer::InsertSwitchNForData(const NodePtr &node, const si REPORT_CALL_ERROR("E19999", "Set Attr:%s to node:%s(%s) failed", ATTR_USER_DESIGNEATE_SHAPE_ORDER.c_str(), switchn_desc->GetName().c_str(), switchn_desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to add user designate shape order attr on switchn node %s", - switchn_desc->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Set][Attr] %s to node:%s(%s) failed", ATTR_USER_DESIGNEATE_SHAPE_ORDER.c_str(), + switchn_desc->GetName().c_str(), switchn_desc->GetType().c_str()); return INTERNAL_ERROR; } if (!AttrUtils::SetBool(switchn_desc, ATTR_INSERT_BY_MBATCH, true)) { REPORT_CALL_ERROR("E19999", "Set Attr:%s to node:%s(%s) failed", ATTR_INSERT_BY_MBATCH.c_str(), switchn_desc->GetName().c_str(), switchn_desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to add insert attr on switchn node %s", switchn_desc->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Set][Attr] %s to node:%s(%s) failed", + ATTR_INSERT_BY_MBATCH.c_str(), switchn_desc->GetName().c_str(), switchn_desc->GetType().c_str()); return INTERNAL_ERROR; } if (!AttrUtils::SetStr(node->GetOpDesc(), kMbatchSwitchnName, switchn_desc->GetName())) { REPORT_CALL_ERROR("E19999", "Set Attr:%s to node:%s(%s) failed", kMbatchSwitchnName, node->GetName().c_str(), node->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to add switchn attr on data node %s", node->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Set][Attr] %s to node:%s(%s) failed", + kMbatchSwitchnName, node->GetName().c_str(), node->GetType().c_str()); return INTERNAL_ERROR; } if (StampDynamicType(switchn_desc) != SUCCESS) { @@ -1506,7 +1539,8 @@ Status MultiBatchGraphCopyer::InsertSwitchNForData(const NodePtr &node, const si REPORT_CALL_ERROR("E19999", "Add node:%s(%s) to graph:%s failed", switchn_desc->GetName().c_str(), switchn_desc->GetType().c_str(), graph_->GetName().c_str()); - GELOGE(OUT_OF_MEMORY, "Failed to create switchn %s from desc", switchn_desc->GetName().c_str()); + GELOGE(OUT_OF_MEMORY, "[Add][Node] %s(%s) to graph:%s failed", + switchn_desc->GetName().c_str(), switchn_desc->GetType().c_str(), graph_->GetName().c_str()); return OUT_OF_MEMORY); if (!getnext_sink_dynamic_dims_) { data_nodes_to_switchn_[node.get()] = switchn; @@ -1553,7 +1587,8 @@ Status MultiBatchGraphCopyer::LinkGetDynamicDimsToNetOutput(const NodePtr &node) if (!AttrUtils::SetStr(node->GetOpDesc(), ATTR_ALL_GEARS_INFO, GetLocalOmgContext().dynamic_dims)) { REPORT_CALL_ERROR("E19999", "Set Attr:%s to node:%s(%s) failed", ATTR_ALL_GEARS_INFO.c_str(), node->GetName().c_str(), node->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to set all gears info attr on netoutput %s.", node->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Set][Attr] %s to node:%s(%s) failed", + ATTR_ALL_GEARS_INFO.c_str(), node->GetName().c_str(), node->GetType().c_str()); return INTERNAL_ERROR; } } @@ -1562,7 +1597,8 @@ Status MultiBatchGraphCopyer::LinkGetDynamicDimsToNetOutput(const NodePtr &node) if (NodeUtils::AppendInputAnchor(node, input_index + 1) != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Append %zu input anchors to node:%s(%s) failed", input_index + 1, node->GetName().c_str(), node->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Append input anchor of %s of %zu failed.", node->GetName().c_str(), input_index); + GELOGE(INTERNAL_ERROR, "[Append][InputAnchor] of node:%s failed, input_index:%zu.", + node->GetName().c_str(), input_index + 1); return INTERNAL_ERROR; } auto ret = @@ -1572,13 +1608,15 @@ Status MultiBatchGraphCopyer::LinkGetDynamicDimsToNetOutput(const NodePtr &node) REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(index:%d) and op:%s(%s)(index:%zu) failed", shape_data_->GetName().c_str(), shape_data_->GetType().c_str(), kDataOutIndex, node->GetName().c_str(), node->GetType().c_str(), input_index); - GELOGE(INTERNAL_ERROR, "Failed to link netoutput %s to getdynamicdims %s", - node->GetName().c_str(), shape_data_->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Add][Edge] between op:%s(%s)(index:%d) and op:%s(%s)(index:%zu) failed", + shape_data_->GetName().c_str(), shape_data_->GetType().c_str(), kDataOutIndex, + node->GetName().c_str(), node->GetType().c_str(), input_index); return INTERNAL_ERROR); if (!AttrUtils::SetBool(node->GetOpDesc(), ATTR_GETNEXT_SINK_DYNMAIC, true)) { REPORT_CALL_ERROR("E19999", "Set Attr:%s to node:%s(%s) failed", ATTR_GETNEXT_SINK_DYNMAIC.c_str(), node->GetName().c_str(), node->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to set getnext sink dynamic attr on netoutput %s.", node->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Set][Attr] %s to node:%s(%s) failed", + ATTR_GETNEXT_SINK_DYNMAIC.c_str(), node->GetName().c_str(), node->GetType().c_str()); return INTERNAL_ERROR; } } @@ -1591,7 +1629,7 @@ Status MultiBatchGraphCopyer::CopyNodeInBatchBranch(const NodePtr &node) { for (size_t i = 0; i < shapes_.size(); ++i) { auto copyed_node = InsertCopyNode(node, i); if (copyed_node == nullptr) { - GELOGE(INTERNAL_ERROR, "Failed to add node to graph when copy node %s", node->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Add][Node] to graph failed, when copy node %s", node->GetName().c_str()); return INTERNAL_ERROR; } copyed_nodes.emplace_back(copyed_node); @@ -1608,7 +1646,8 @@ Status MultiBatchGraphCopyer::AddAttrForGetDynamicDims(const NodePtr &node) { REPORT_CALL_ERROR("E19999", "Set Attr:%s to node:%s(%s) failed", ATTR_GETNEXT_SINK_DATA_COUNT.c_str(), shape_data_->GetName().c_str(), shape_data_->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "set ATTR_GETNEXT_SINK_DATA_COUNT failed"); + GELOGE(INTERNAL_ERROR, "[Set][Attr] %s to node:%s(%s) failed", ATTR_GETNEXT_SINK_DATA_COUNT.c_str(), + shape_data_->GetName().c_str(), shape_data_->GetType().c_str()); return INTERNAL_ERROR; } vector shape_info; @@ -1627,7 +1666,8 @@ Status MultiBatchGraphCopyer::AddAttrForGetDynamicDims(const NodePtr &node) { REPORT_CALL_ERROR("E19999", "Set Attr:%s to node:%s(%s) failed", ATTR_GETNEXT_SINK_SHAPE_INFO.c_str(), shape_data_->GetName().c_str(), shape_data_->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "set ATTR_GETNEXT_SINK_SHAPE_INFO failed"); + GELOGE(INTERNAL_ERROR, "[Set][Attr] %s to node:%s(%s) failed", ATTR_GETNEXT_SINK_SHAPE_INFO.c_str(), + shape_data_->GetName().c_str(), shape_data_->GetType().c_str()); return INTERNAL_ERROR; } return SUCCESS; @@ -1643,14 +1683,14 @@ Status MultiBatchGraphCopyer::AddLinkForGetDynamicDims(const NodePtr &node) { shape_data_->GetName().c_str(), input_index); auto out_data_anchor = node->GetOutDataAnchor(out_index); auto ret = GraphUtils::AddEdge(out_data_anchor, shape_data_->GetInDataAnchor(input_index)); - GE_IF_BOOL_EXEC( - ret != GRAPH_SUCCESS, - REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(index:%zu) and op:%s(%s)(index:%zu) failed", - node->GetName().c_str(), node->GetType().c_str(), out_index, - shape_data_->GetName().c_str(), shape_data_->GetType().c_str(), input_index); - GELOGE(INTERNAL_ERROR, "Failed to link getnext %s to getdynamicdims %s", - node->GetName().c_str(), shape_data_->GetName().c_str()); - return INTERNAL_ERROR); + GE_IF_BOOL_EXEC(ret != GRAPH_SUCCESS, + REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(index:%zu) and op:%s(%s)(index:%zu) failed", + node->GetName().c_str(), node->GetType().c_str(), out_index, + shape_data_->GetName().c_str(), shape_data_->GetType().c_str(), input_index); + GELOGE(INTERNAL_ERROR, "[Add][Edge] between op:%s(%s)(index:%zu) and op:%s(%s)(index:%zu) failed", + node->GetName().c_str(), node->GetType().c_str(), out_index, + shape_data_->GetName().c_str(), shape_data_->GetType().c_str(), input_index); + return INTERNAL_ERROR); } return SUCCESS; } @@ -1666,7 +1706,8 @@ Status MultiBatchGraphCopyer::LinkEdges() { REPORT_INNER_ERROR("E19999", "swithn in data_nodes_to_switchn_ for op:%s(%s) is nullptr, check invalid", node->GetName().c_str(), node->GetType().c_str()); - GELOGE(PARAM_INVALID, "Switchn should not be nullptr for %s.", node->GetName().c_str()); + GELOGE(PARAM_INVALID, "[Check][Param]Switchn should not be nullptr for %s.", + node->GetName().c_str()); return OUT_OF_MEMORY); ret = LinkDataToSwitchN(node, switchn, kDataOutIndex); GE_CHK_STATUS_RET(ret, "Link data to switchn failed."); @@ -1674,21 +1715,21 @@ Status MultiBatchGraphCopyer::LinkEdges() { } else { if (IsGetNextType(node)) { GELOGD("Start add attr and link edge for %s.", node->GetName().c_str()); - GE_CHK_STATUS_RET(AddAttrForGetDynamicDims(node), "Failed to add attr for %s.", node->GetName().c_str()); - GE_CHK_STATUS_RET(AddLinkForGetDynamicDims(node), "Failed to add link for %s.", node->GetName().c_str()); + GE_CHK_STATUS_RET(AddAttrForGetDynamicDims(node), "[Add][Attr] for %s failed.", node->GetName().c_str()); + GE_CHK_STATUS_RET(AddLinkForGetDynamicDims(node), "[Add][Link] for %s failed.", node->GetName().c_str()); } for (size_t i = 0; i < getnext_nodes_to_switchn_.size(); ++i) { for (size_t j = 0; j < getnext_nodes_to_switchn_.at(i).size(); ++j) { if (getnext_nodes_to_switchn_.at(i).at(j).first == node.get()) { auto switchn = getnext_nodes_to_switchn_.at(i).at(j).second; - GE_CHK_STATUS_RET(LinkDataToSwitchN(node, switchn, i), "Link %s to %s failed.", node->GetName().c_str(), - switchn->GetName().c_str()); + GE_CHK_STATUS_RET(LinkDataToSwitchN(node, switchn, i), + "[Link][Data] %s to %s failed.", node->GetName().c_str(), switchn->GetName().c_str()); } } } } if (nodes_to_merge_nodes_.count(node.get()) > 0) { - GE_CHK_STATUS_RET(LinkToMerge(node), "Link %s to merge failed.", node->GetName().c_str()); + GE_CHK_STATUS_RET(LinkToMerge(node), "[Link][Node] %s to merge failed.", node->GetName().c_str()); } if (nodes_to_batch_nodes_.count(node.get()) > 0) { ret = LinkToNodeInBranch(node); @@ -1705,24 +1746,24 @@ Status MultiBatchGraphCopyer::LinkEdges() { Status MultiBatchGraphCopyer::LinkDataToSwitchN(const NodePtr &data, const NodePtr &switchn, const int &out_index) { auto ret = GraphUtils::AddEdge(shape_data_->GetOutDataAnchor(kDataOutIndex), switchn->GetInDataAnchor(kSwitchNPredIndex)); - GE_IF_BOOL_EXEC( - ret != GRAPH_SUCCESS, - REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(index:%d) and op:%s(%s)(index:%d) failed", - shape_data_->GetName().c_str(), shape_data_->GetType().c_str(), kDataOutIndex, - switchn->GetName().c_str(), switchn->GetType().c_str(), kSwitchNPredIndex); - GELOGE(INTERNAL_ERROR, "Failed to link shape data %s to switchn %s", - shape_data_->GetName().c_str(), switchn->GetName().c_str()); - return INTERNAL_ERROR); + GE_IF_BOOL_EXEC(ret != GRAPH_SUCCESS, + REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(index:%d) and op:%s(%s)(index:%d) failed", + shape_data_->GetName().c_str(), shape_data_->GetType().c_str(), kDataOutIndex, + switchn->GetName().c_str(), switchn->GetType().c_str(), kSwitchNPredIndex); + GELOGE(INTERNAL_ERROR, "[Add][Edge] between op:%s(%s)(index:%d) and op:%s(%s)(index:%d) failed", + shape_data_->GetName().c_str(), shape_data_->GetType().c_str(), kDataOutIndex, + switchn->GetName().c_str(), switchn->GetType().c_str(), kSwitchNPredIndex); + return INTERNAL_ERROR); ret = GraphUtils::AddEdge(data->GetOutDataAnchor(out_index), switchn->GetInDataAnchor(kSwitchNDataIndex)); - GE_IF_BOOL_EXEC( - ret != GRAPH_SUCCESS, - REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(index:%d) and op:%s(%s)(index:%d) failed", - data->GetName().c_str(), data->GetType().c_str(), out_index, - switchn->GetName().c_str(), switchn->GetType().c_str(), kSwitchNDataIndex); - GELOGE(INTERNAL_ERROR, "Failed to link data %s to switchn %s", - data->GetName().c_str(), switchn->GetName().c_str()); - return INTERNAL_ERROR); + GE_IF_BOOL_EXEC(ret != GRAPH_SUCCESS, + REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(index:%d) and op:%s(%s)(index:%d) failed", + data->GetName().c_str(), data->GetType().c_str(), out_index, + switchn->GetName().c_str(), switchn->GetType().c_str(), kSwitchNDataIndex); + GELOGE(INTERNAL_ERROR, "[Add][Edge] between op:%s(%s)(index:%d) and op:%s(%s)(index:%d) failed", + data->GetName().c_str(), data->GetType().c_str(), out_index, + switchn->GetName().c_str(), switchn->GetType().c_str(), kSwitchNDataIndex); + return INTERNAL_ERROR); return SUCCESS; } @@ -1766,7 +1807,7 @@ Status MultiBatchGraphCopyer::LinkToMerge(const NodePtr &node) { } REPORT_INNER_ERROR("E19999", "The merge node %s is created, index %zu, but can not find the src node, " "check invalid", merge_node->GetName().c_str(), i); - GELOGE(INTERNAL_ERROR, "The merge node %s is created, index %zu, but can not find the src node", + GELOGE(INTERNAL_ERROR, "[Check][Param] The merge node %s is created, index %zu, but can not find the src node", merge_node->GetName().c_str(), i); return INTERNAL_ERROR; } @@ -1806,7 +1847,7 @@ Status MultiBatchGraphCopyer::LinkToNodeOutBranch(const NodePtr &node) { "cause no merge node found, check invalid", in_node->GetName().c_str(), in_node->GetType().c_str(), src_out_anchor->GetIdx(), node->GetName().c_str(), node->GetType().c_str(), in_data_anchor->GetIdx()); - GELOGE(INTERNAL_ERROR, "Failed to link IO data edge from %s(%d) to %s(%d), no merge node found", + GELOGE(INTERNAL_ERROR, "[Check][Param] Failed to link IO data edge from %s(%d) to %s(%d), no merge node found", in_node->GetName().c_str(), src_out_anchor->GetIdx(), node->GetName().c_str(), in_data_anchor->GetIdx()); return INTERNAL_ERROR; } @@ -1816,7 +1857,7 @@ Status MultiBatchGraphCopyer::LinkToNodeOutBranch(const NodePtr &node) { "cause no merge node found, check invalid", in_node->GetName().c_str(), in_node->GetType().c_str(), src_out_anchor->GetIdx(), node->GetName().c_str(), node->GetType().c_str(), in_data_anchor->GetIdx()); - GELOGE(INTERNAL_ERROR, "Failed to link IO data edge from %s(%d) to %s(%d), no merge node found", + GELOGE(INTERNAL_ERROR, "[Check][Param] Failed to link IO data edge from %s(%d) to %s(%d), no merge node found", in_node->GetName().c_str(), src_out_anchor->GetIdx(), node->GetName().c_str(), in_data_anchor->GetIdx()); return INTERNAL_ERROR; } @@ -1825,8 +1866,9 @@ Status MultiBatchGraphCopyer::LinkToNodeOutBranch(const NodePtr &node) { REPORT_INNER_ERROR("E19999", "Unlink edge from %s(%s)(index:%d) to %s(%s)(index:%d) failed", in_node->GetName().c_str(), in_node->GetType().c_str(), src_out_anchor->GetIdx(), node->GetName().c_str(), node->GetType().c_str(), in_data_anchor->GetIdx()); - GELOGE(INTERNAL_ERROR, "Failed to unlink the control edge from %s(%d) to %s(%d)", in_node->GetName().c_str(), - src_out_anchor->GetIdx(), node->GetName().c_str(), in_data_anchor->GetIdx()); + GELOGE(INTERNAL_ERROR, "[Unlink][Edge] from %s(%s)(index:%d) to %s(%s)(index:%d) failed", + in_node->GetName().c_str(), in_node->GetType().c_str(), src_out_anchor->GetIdx(), + node->GetName().c_str(), node->GetType().c_str(), in_data_anchor->GetIdx()); return INTERNAL_ERROR; } ret = GraphUtils::AddEdge(merge_node->GetOutDataAnchor(kMergeDataOutIndex), in_data_anchor); @@ -1834,8 +1876,9 @@ Status MultiBatchGraphCopyer::LinkToNodeOutBranch(const NodePtr &node) { REPORT_CALL_ERROR("E19999", "Add edge between op:%s(%s)(index:%d) and op:%s(%s)(index:%d) failed", merge_node->GetName().c_str(), merge_node->GetType().c_str(), kMergeDataOutIndex, node->GetName().c_str(), node->GetType().c_str(), in_data_anchor->GetIdx()); - GELOGE(INTERNAL_ERROR, "Failed to add data edge from %s(%d) to %s(%d)", merge_node->GetName().c_str(), - src_out_anchor->GetIdx(), node->GetName().c_str(), in_data_anchor->GetIdx()); + GELOGE(INTERNAL_ERROR, "[Add][Edge] between op:%s(%s)(index:%d) and op:%s(%s)(index:%d) failed", + merge_node->GetName().c_str(), merge_node->GetType().c_str(), kMergeDataOutIndex, + node->GetName().c_str(), node->GetType().c_str(), in_data_anchor->GetIdx()); return INTERNAL_ERROR; } GELOGI("Link data edge from merge %s(from %s(%d)) to %s(%d)", merge_node->GetName().c_str(), @@ -1852,7 +1895,7 @@ Status MultiBatchGraphCopyer::LinkToNodeOutBranch(const NodePtr &node) { "check invalid", in_node->GetName().c_str(), in_node->GetType().c_str(), node->GetName().c_str(), node->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to link IO control edge from %s to %s, no merge node found", + GELOGE(INTERNAL_ERROR, "[Check][Param] Failed to link IO control edge from %s to %s, no merge node found", in_node->GetName().c_str(), node->GetName().c_str()); return INTERNAL_ERROR; } @@ -1862,7 +1905,7 @@ Status MultiBatchGraphCopyer::LinkToNodeOutBranch(const NodePtr &node) { "Failed to link IO control edge from %s(%s) to %s(%s), no merge node found, check invalid", in_node->GetName().c_str(), in_node->GetType().c_str(), node->GetName().c_str(), node->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to link IO control edge from %s to %s, no merge node found", + GELOGE(INTERNAL_ERROR, "[Check][Param] Failed to link IO control edge from %s to %s, no merge node found", in_node->GetName().c_str(), node->GetName().c_str()); return INTERNAL_ERROR; } @@ -1870,15 +1913,17 @@ Status MultiBatchGraphCopyer::LinkToNodeOutBranch(const NodePtr &node) { GE_IF_BOOL_EXEC(in_node->GetOutControlAnchor() == nullptr, REPORT_INNER_ERROR("E19999", "Out control anchor of op:%s(%s) is nullptr, check invalid", in_node->GetName().c_str(), in_node->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Innode outputControlAnchor is null"); + GELOGE(INTERNAL_ERROR, "[Get][OutControlAnchor]Out control anchor of op:%s(%s) is nullptr", + in_node->GetName().c_str(), in_node->GetType().c_str()); return INTERNAL_ERROR); auto ret = in_node->GetOutControlAnchor()->Unlink(node->GetInControlAnchor()); GE_IF_BOOL_EXEC(ret != GRAPH_SUCCESS, REPORT_INNER_ERROR("E19999", "Unlink ctrl edge from %s(%s) to %s(%s) failed", in_node->GetName().c_str(), in_node->GetType().c_str(), node->GetName().c_str(), node->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to unlink the control edge from %s to %s", - in_node->GetName().c_str(), node->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Unlink][CtrlEdge] from %s(%s) to %s(%s) failed", + in_node->GetName().c_str(), in_node->GetType().c_str(), + node->GetName().c_str(), node->GetType().c_str()); return INTERNAL_ERROR); ret = GraphUtils::AddEdge(merge_node->GetOutControlAnchor(), node->GetInControlAnchor()); GE_IF_BOOL_EXEC( @@ -1886,8 +1931,9 @@ Status MultiBatchGraphCopyer::LinkToNodeOutBranch(const NodePtr &node) { REPORT_CALL_ERROR("E19999", "Add ctrl edge between op:%s(%s) and op:%s(%s) failed", merge_node->GetName().c_str(), merge_node->GetType().c_str(), node->GetName().c_str(), node->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to add control edge from %s to %s", - merge_node->GetName().c_str(), node->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Add][CtrlEdge] between op:%s(%s) and op:%s(%s) failed", + merge_node->GetName().c_str(), merge_node->GetType().c_str(), + node->GetName().c_str(), node->GetType().c_str()); return INTERNAL_ERROR); GELOGI("Link control edge from merge %s(from %s) to %s", merge_node->GetName().c_str(), in_node->GetName().c_str(), node->GetName().c_str()); @@ -1911,15 +1957,15 @@ Status ProcessMultiBatch(ComputeGraphPtr &graph) { std::vector getnext_nosink_nodes; std::vector getnext_sink_nodes; if (CheckSequenceOfOptions(graph, data_nodes, getnext_nosink_nodes, getnext_sink_nodes) != SUCCESS) { - GELOGE(PARAM_INVALID, "[Train_Dynamic] CheckSequenceOfOptions failed."); + GELOGE(PARAM_INVALID, "[Train_Dynamic][Check][SequenceOfOptions] failed."); return PARAM_INVALID; } if (UpdateNameOfInputShape(graph, data_nodes, getnext_nosink_nodes, getnext_sink_nodes) != SUCCESS) { - GELOGE(PARAM_INVALID, "[Train_Dynamic] UpdateNameForInputShapeOfOption failed."); + GELOGE(PARAM_INVALID, "[Train_Dynamic][Update][NameOfInputShape] failed."); return PARAM_INVALID; } if (DeleteIdentityInsertByAdapter(graph) != SUCCESS) { - GELOGE(PARAM_INVALID, "DeleteIdentityInsertByAdapter failed."); + GELOGE(PARAM_INVALID, "[Call][DeleteIdentityInsertByAdapter] failed."); return PARAM_INVALID; } @@ -1929,7 +1975,7 @@ Status ProcessMultiBatch(ComputeGraphPtr &graph) { return SUCCESS; } if (CheckNegativeCountOfOptions(shapes) != SUCCESS) { - GELOGE(PARAM_INVALID, "Input_shape and dynamic_dims should set correct params."); + GELOGE(PARAM_INVALID, "[Check][Param] Input_shape and dynamic_dims should set correct params."); return PARAM_INVALID; } @@ -1986,7 +2032,8 @@ void GetDynamicShapeByGraph(const ComputeGraphPtr &graph, const NodePtr &node, if (subgraph == nullptr) { REPORT_INNER_ERROR("E19999", "Get subgraph:%s from graph:%s failed", dynamic_branch_names[j].c_str(), graph->GetName().c_str()); - GELOGE(GE_GRAPH_EMPTY_SUBGRAPH, "Subgraph not found, name: %s", dynamic_branch_names[j].c_str()); + GELOGE(GE_GRAPH_EMPTY_SUBGRAPH, "[Get][SubGraph] %s from graph:%s failed", + dynamic_branch_names[j].c_str(), graph->GetName().c_str()); dynamic_output_dims.clear(); return; } @@ -1995,7 +2042,8 @@ void GetDynamicShapeByGraph(const ComputeGraphPtr &graph, const NodePtr &node, if (out_node == nullptr) { REPORT_INNER_ERROR("E19999", "No netoutput node exist in subgraph:%s, check invalid", subgraph->GetName().c_str()); - GELOGE(GE_GRAPH_GRAPH_NODE_NULL, "NetOutput not found, name: %s", dynamic_branch_names[j].c_str()); + GELOGE(GE_GRAPH_GRAPH_NODE_NULL, "[Check][Param] No netoutput node exist in subgraph:%s", + subgraph->GetName().c_str()); dynamic_output_dims.clear(); return; } @@ -2006,7 +2054,9 @@ void GetDynamicShapeByGraph(const ComputeGraphPtr &graph, const NodePtr &node, REPORT_INNER_ERROR("E19999", "op_desc of node in subgraph:%s is nullptr or input desc size:%zu <= %zu, check invalid", subgraph->GetName().c_str(), out_desc->GetInputsSize(), i); - GELOGE(GE_GRAPH_GRAPH_NODE_NULL, "Get Input desc failed, name: %s, index: %zu", out_node->GetName().c_str(), i); + GELOGE(GE_GRAPH_GRAPH_NODE_NULL, + "[Check][Param] op_desc of node in subgraph:%s is nullptr or input desc size:%zu <= %zu", + subgraph->GetName().c_str(), out_desc->GetInputsSize(), i); dynamic_output_dims.clear(); return; } @@ -2110,7 +2160,8 @@ Status GetDynamicOutputShape(ComputeGraphPtr &graph) { REPORT_CALL_ERROR("E19999", "Set Attr:%s to node:%s(%s) failed", ATTR_NAME_DYNAMIC_OUTPUT_DIMS.c_str(), net_output->GetName().c_str(), net_output->GetType().c_str()); - GELOGE(FAILED, "Set dynamic output dims attr failed"); + GELOGE(FAILED, "[Set][Attr] %s to node:%s(%s) failed", ATTR_NAME_DYNAMIC_OUTPUT_DIMS.c_str(), + net_output->GetName().c_str(), net_output->GetType().c_str()); return FAILED; } } diff --git a/ge/graph/preprocess/multi_batch_options.cc b/ge/graph/preprocess/multi_batch_options.cc index b82d1034..058b720d 100644 --- a/ge/graph/preprocess/multi_batch_options.cc +++ b/ge/graph/preprocess/multi_batch_options.cc @@ -105,8 +105,9 @@ Status CheckSequenceOfData(ComputeGraphPtr &graph, const vector &data_n REPORT_INNER_ERROR("E19999", "Count:%zu of data_nodes in graph:%s should be equal to " "input_shape count:%zu from option, check invalid", data_nodes.size(), graph->GetName().c_str(), GetLocalOmgContext().user_input_dims.size()); - GELOGE(PARAM_INVALID, "The count of input shape:%zu should be equal to the count of data num:%zu.", - GetLocalOmgContext().user_input_dims.size(), data_nodes.size()); + GELOGE(PARAM_INVALID, "[Check][Param] Count:%zu of data_nodes in graph:%s should be equal to " + "input_shape count:%zu from option", + data_nodes.size(), graph->GetName().c_str(), GetLocalOmgContext().user_input_dims.size()); return PARAM_INVALID; } for (size_t i = 0; i < data_nodes.size(); ++i) { @@ -127,9 +128,11 @@ Status CheckSequenceOfData(ComputeGraphPtr &graph, const vector &data_n formats::JoinToString(output_shape).c_str(), GetLocalOmgContext().user_input_dims.at(i).first.c_str(), formats::JoinToString(dynamic_dims).c_str(), graph->GetName().c_str()); - GELOGE(PARAM_INVALID, "The output shape of %s is %s, the input shape from options of %s is %s.", + GELOGE(PARAM_INVALID, "[Check][Param] The output shape of %s is %s, " + "the input shape from options of %s is %s, graph:%s", data_node->GetName().c_str(), formats::JoinToString(output_shape).c_str(), - GetLocalOmgContext().user_input_dims.at(i).first.c_str(), formats::JoinToString(dynamic_dims).c_str()); + GetLocalOmgContext().user_input_dims.at(i).first.c_str(), + formats::JoinToString(dynamic_dims).c_str(), graph->GetName().c_str()); return PARAM_INVALID; } for (size_t j = 0; j < dynamic_dims.size(); ++j) { @@ -139,8 +142,10 @@ Status CheckSequenceOfData(ComputeGraphPtr &graph, const vector &data_n formats::JoinToString(dynamic_dims).c_str(), formats::JoinToString(output_shape).c_str(), data_node->GetName().c_str(), kDynmaicDims, j, graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "Value of input shape %s should be equal to %s.", - formats::JoinToString(dynamic_dims).c_str(), formats::JoinToString(output_shape).c_str()); + GELOGE(INTERNAL_ERROR, "[Check][Param] Value of input shape %s from option and output shape %s of data op:%s " + "should be equal to %d, index:%zu, graph:%s", + formats::JoinToString(dynamic_dims).c_str(), formats::JoinToString(output_shape).c_str(), + data_node->GetName().c_str(), kDynmaicDims, j, graph->GetName().c_str()); return INTERNAL_ERROR; } } @@ -154,7 +159,8 @@ Status CheckSequenceOfGetnext(ComputeGraphPtr &graph, const vector &get REPORT_INNER_ERROR("E19999", "Not support dynamic dims when a graph with multi getnext nodes, graph:%s, " "num of getnext node:%zu, check invalid", graph->GetName().c_str(), getnext_sink_node.size()); - GELOGE(PARAM_INVALID, "Not support dynamic dims when a graph with multi getnext nodes."); + GELOGE(PARAM_INVALID, "[Check][Param] Not support dynamic dims when a graph with multi getnext nodes, graph:%s, " + "num of getnext node:%zu", graph->GetName().c_str(), getnext_sink_node.size()); return PARAM_INVALID; } auto data_node = getnext_sink_node.at(0); @@ -163,11 +169,12 @@ Status CheckSequenceOfGetnext(ComputeGraphPtr &graph, const vector &get GE_CHECK_NOTNULL(op_desc); size_t data_count = data_node->GetAllOutDataAnchors().size() / kDivisionConst; if (data_count != GetLocalOmgContext().user_input_dims.size()) { - REPORT_INNER_ERROR("E19999", "Output desc count of %s is %zu, should be equal to count of input shape: %zu, " + REPORT_INNER_ERROR("E19999", "Output desc count of %s is %zu, should be equal to count of input shape:%zu, " "graph:%s, check invalid", op_desc->GetName().c_str(), data_count, GetLocalOmgContext().user_input_dims.size(), graph->GetName().c_str()); - GELOGE(PARAM_INVALID, "Output count of %s is %zu, should be equal to count of input shape: %zu", - op_desc->GetName().c_str(), data_count, GetLocalOmgContext().user_input_dims.size()); + GELOGE(PARAM_INVALID, "[Check][Param] Output desc count of %s is %zu, " + "should be equal to count of input shape:%zu, graph:%s", op_desc->GetName().c_str(), + data_count, GetLocalOmgContext().user_input_dims.size(), graph->GetName().c_str()); return PARAM_INVALID; } for (size_t i = 0; i < data_count; ++i) { @@ -186,9 +193,10 @@ Status CheckSequenceOfGetnext(ComputeGraphPtr &graph, const vector &get formats::JoinToString(dynamic_dims).c_str(), GetLocalOmgContext().user_input_dims.at(i).first.c_str(), graph->GetName().c_str()); - GELOGE(PARAM_INVALID, "the output_shape of %s is %s, the input_shape from options of %s is %s.", - data_node->GetName().c_str(), formats::JoinToString(output_shape).c_str(), - GetLocalOmgContext().user_input_dims.at(i).first.c_str(), formats::JoinToString(dynamic_dims).c_str()); + GELOGE(PARAM_INVALID, "[Check][Param] The %zu output_shape of %s is %s not equal to the input_shape:%s " + "from options of %s, graph:%s", i, data_node->GetName().c_str(), + formats::JoinToString(output_shape).c_str(), formats::JoinToString(dynamic_dims).c_str(), + GetLocalOmgContext().user_input_dims.at(i).first.c_str(), graph->GetName().c_str()); return PARAM_INVALID; } for (size_t j = 0; j < dynamic_dims.size(); ++j) { @@ -198,8 +206,10 @@ Status CheckSequenceOfGetnext(ComputeGraphPtr &graph, const vector &get formats::JoinToString(dynamic_dims).c_str(), formats::JoinToString(output_shape).c_str(), data_node->GetName().c_str(), kDynmaicDims, j, graph->GetName().c_str()); - GELOGE(INTERNAL_ERROR, "value of input_shape %s should be equal to %s.", - formats::JoinToString(dynamic_dims).c_str(), formats::JoinToString(output_shape).c_str()); + GELOGE(INTERNAL_ERROR, "[Check][Param] Value of input shape %s from option and output shape %s of data op:%s " + "should be equal to %d, index:%zu, graph:%s", formats::JoinToString(dynamic_dims).c_str(), + formats::JoinToString(output_shape).c_str(), data_node->GetName().c_str(), kDynmaicDims, + j, graph->GetName().c_str()); return INTERNAL_ERROR; } } @@ -215,26 +225,26 @@ Status CheckSequenceOfOptions(ComputeGraphPtr &graph, vector &data_node } if (DistinguishGetNextAndData(graph, data_nodes, getnext_nosink_nodes, getnext_sink_nodes) != SUCCESS) { - GELOGE(PARAM_INVALID, "DistinguishGetNextAndData failed."); + GELOGE(PARAM_INVALID, "[Call][DistinguishGetNextAndData] failed."); return PARAM_INVALID; } if (GetLocalOmgContext().dynamic_node_type == DATA) { GELOGD("Users want data nodes to be dynamic."); if (CheckSequenceOfData(graph, data_nodes) != SUCCESS) { - GELOGE(PARAM_INVALID, "Failed to check sequence of data nodes."); + GELOGE(PARAM_INVALID, "[Check][Sequence] Of Data nodes failed."); return PARAM_INVALID; } } else { GELOGD("Users want getnext nodes to be dynamic."); if (!getnext_nosink_nodes.empty()) { if (CheckSequenceOfData(graph, getnext_nosink_nodes) != SUCCESS) { - GELOGE(PARAM_INVALID, "Failed to check sequence of getnext nosink nodes."); + GELOGE(PARAM_INVALID, "[Check][Sequence] of getnext nosink nodes failed."); return PARAM_INVALID; } } else { if (CheckSequenceOfGetnext(graph, getnext_sink_nodes) != SUCCESS) { - GELOGE(PARAM_INVALID, "Failed to check sequence of getnext sink nodes."); + GELOGE(PARAM_INVALID, "[Check][Sequence] of getnext sink nodes failed."); return PARAM_INVALID; } } @@ -248,8 +258,9 @@ Status UpdateNameOfData(ComputeGraphPtr &graph, const vector &data_node REPORT_INNER_ERROR("E19999", "Count:%zu of data_nodes in graph:%s should be equal to " "input_shape count:%zu from option, check invalid", data_nodes.size(), graph->GetName().c_str(), GetLocalOmgContext().user_input_dims.size()); - GELOGE(PARAM_INVALID, "count of data_nodes: %zu should be equal to input_shape count: %zu.", - data_nodes.size(), GetLocalOmgContext().user_input_dims.size()); + GELOGE(PARAM_INVALID, "[Check][Param] Count:%zu of data_nodes in graph:%s should be equal to " + "input_shape count:%zu from option", + data_nodes.size(), graph->GetName().c_str(), GetLocalOmgContext().user_input_dims.size()); return PARAM_INVALID; } for (size_t i = 0; i < data_nodes.size(); ++i) { @@ -265,7 +276,8 @@ Status UpdateNameOfGetnext(ComputeGraphPtr &graph, const vector &getnex REPORT_INNER_ERROR("E19999", "Not support dynamic dims when a graph with multi getnext nodes, graph:%s, " "num of getnext node:%zu, check invalid", graph->GetName().c_str(), getnext_sink_nodes.size()); - GELOGE(PARAM_INVALID, "Not support dynamic dims when a graph with multi getnext nodes."); + GELOGE(PARAM_INVALID, "[Check][Param] Not support dynamic dims when a graph with multi getnext nodes, graph:%s, " + "num of getnext node:%zu", graph->GetName().c_str(), getnext_sink_nodes.size()); return PARAM_INVALID; } auto input_node = getnext_sink_nodes.at(0); @@ -275,11 +287,12 @@ Status UpdateNameOfGetnext(ComputeGraphPtr &graph, const vector &getnex // user want getnext dynamic, just getnext or data+getnext_sink size_t data_count = input_node->GetAllOutDataAnchors().size() / kDivisionConst; if (data_count != GetLocalOmgContext().user_input_dims.size()) { - REPORT_INNER_ERROR("E19999", "Output desc count of %s is %zu, should be equal to count of input shape: %zu, " + REPORT_INNER_ERROR("E19999", "Output desc count of %s is %zu, should be equal to count of input shape:%zu, " "graph:%s, check invalid", op_desc->GetName().c_str(), data_count, GetLocalOmgContext().user_input_dims.size(), graph->GetName().c_str()); - GELOGE(PARAM_INVALID, "Output count of %s is %zu, should be equal to count of input shape: %zu", - op_desc->GetName().c_str(), data_count, GetLocalOmgContext().user_input_dims.size()); + GELOGE(PARAM_INVALID, "[Check][Param]Output desc count of %s is %zu, " + "should be equal to count of input shape:%zu, graph:%s", op_desc->GetName().c_str(), data_count, + GetLocalOmgContext().user_input_dims.size(), graph->GetName().c_str()); return PARAM_INVALID; } @@ -302,19 +315,21 @@ Status UpdateNameOfInputShape(ComputeGraphPtr &graph, const vector &dat if (GetLocalOmgContext().dynamic_node_type == DATA) { GELOGD("Users want data nodes to be dynamic."); if (UpdateNameOfData(graph, data_nodes) != SUCCESS) { - GELOGE(PARAM_INVALID, "Failed to update first value of input shape of data nodes."); + GELOGE(PARAM_INVALID, "[Call][UpdateNameOfData] update first value of input shape of data nodes failed."); return PARAM_INVALID; } } else { GELOGD("Users want getnext nodes to be dynamic."); if (!getnext_nosink_nodes.empty()) { if (UpdateNameOfData(graph, getnext_nosink_nodes) != SUCCESS) { - GELOGE(PARAM_INVALID, "Failed to update first value of input shape of getnext nosink nodes."); + GELOGE(PARAM_INVALID, + "[Call][UpdateNameOfData] update first value of input shape of getnext nosink nodes failed."); return PARAM_INVALID; } } else { if (UpdateNameOfGetnext(graph, getnext_sink_nodes) != SUCCESS) { - GELOGE(PARAM_INVALID, "Failed to update first value of input shape of getnext sink nodes."); + GELOGE(PARAM_INVALID, + "[Call][UpdateNameOfGetnext] update first value of input shape of getnext sink nodes failed."); return PARAM_INVALID; } } @@ -335,12 +350,13 @@ Status DeleteIdentityInsertByAdapter(ComputeGraphPtr &graph) { GE_IF_BOOL_EXEC(peer_in_anchor == nullptr, continue); auto dst_node = peer_in_anchor->GetOwnerNode(); GE_IF_BOOL_EXEC(dst_node == nullptr, continue); - if (dst_node->GetType() == IDENTITY) { + if (dst_node->GetType() == IDENTITY && dst_node->GetOutDataNodes().empty()) { GELOGI("Need to remove %s.", dst_node->GetName().c_str()); - if (ge::GraphUtils::RemoveNodeWithoutRelink(graph, dst_node) != GRAPH_SUCCESS) { + if (GraphUtils::RemoveNodeWithoutRelink(graph, dst_node) != GRAPH_SUCCESS) { REPORT_CALL_ERROR("E19999", "Remove node:%s(%s) from graph:%s failed", dst_node->GetName().c_str(), dst_node->GetType().c_str(), graph->GetName().c_str()); - GELOGE(FAILED, "Remove Identity node %s failed.", dst_node->GetName().c_str()); + GELOGE(FAILED, "[Remove][Node] %s(%s) from graph:%s failed", + dst_node->GetName().c_str(), dst_node->GetType().c_str(), graph->GetName().c_str()); return FAILED; } } @@ -365,8 +381,8 @@ Status CheckNegativeCountOfOptions(const std::vector> &shap if (shapes.at(i).size() != negative_count) { REPORT_INNER_ERROR("E19999", "gear num of dynamic_dims is %zu should be equal to num:%zu from option, " "check invalid", shapes.at(i).size(), negative_count); - GELOGE(PARAM_INVALID, "Each gear num of dynamic_dims is %zu should be equal to %zu.", shapes.at(i).size(), - negative_count); + GELOGE(PARAM_INVALID, "[Check][Param] gear num of dynamic_dims is %zu should be equal to num:%zu from option", + shapes.at(i).size(), negative_count); return PARAM_INVALID; } } @@ -443,7 +459,7 @@ Status ParserDataToDynamicInfo(const vector> &shapes, if (tmp_index >= dynamic_gear_info.size()) { ErrorManager::GetInstance().ATCReportErrMessage( "E10045", {"name", "shape"}, {data_name, formats::JoinToString(data_shape)}); - GELOGE(PARAM_INVALID, "Data: %s shape: %s make dynamic dims overflow", data_name.c_str(), + GELOGE(PARAM_INVALID, "[Check][Param] Data:%s shape:%s make dynamic dims overflow", data_name.c_str(), formats::JoinToString(data_shape).c_str()); return FAILED; } @@ -452,7 +468,8 @@ Status ParserDataToDynamicInfo(const vector> &shapes, } else { ErrorManager::GetInstance().ATCReportErrMessage( "E10046", {"name", "shape"}, {data_name, formats::JoinToString(data_shape)}); - GELOGE(PARAM_INVALID, "Dynamic dims num of data: %s shape: %s can not be more than one gear dynamic info size", + GELOGE(PARAM_INVALID, "[Check][Param] Dynamic dims num of data: %s shape: %s " + "can not be more than one gear dynamic info size", data_name.c_str(), formats::JoinToString(data_shape).c_str()); return FAILED; } @@ -477,7 +494,7 @@ Status CheckDynamicParams(const vector> &shapes) { ErrorManager::GetInstance().ATCReportErrMessage( "E10035", {"shapesize", "minshapesize"}, {std::to_string(shapes.size()), std::to_string(kMinShapesCount - 1)}); GELOGE(PARAM_INVALID, - "Input parameter[--dynamic_batch_size, --dynamic_image_size or --dynamic_dims]'s " + "[Check][Param] Input parameter[--dynamic_batch_size, --dynamic_image_size or --dynamic_dims]'s " "value size [%zu] must be greater than [%d].", shapes.size(), kMinShapesCount - 1); return PARAM_INVALID; @@ -486,7 +503,7 @@ Status CheckDynamicParams(const vector> &shapes) { ErrorManager::GetInstance().ATCReportErrMessage( "E10036", {"shapesize", "maxshapesize"}, {std::to_string(shapes.size()), std::to_string(kMaxShapesCount + 1)}); GELOGE(PARAM_INVALID, - "Input parameter[--dynamic_batch_size, --dynamic_image_size or --dynamic_dims]'s " + "[Check][Param] Input parameter[--dynamic_batch_size, --dynamic_image_size or --dynamic_dims]'s " "value size [%zu] must be less than [%d].", shapes.size(), kMaxShapesCount + 1); return PARAM_INVALID; @@ -498,7 +515,7 @@ Status CheckDynamicParams(const vector> &shapes) { ErrorManager::GetInstance().ATCReportErrMessage("E10037", {"shapesize1", "shapesize2"}, {std::to_string(shape_size), std::to_string(shape.size())}); GELOGE(PARAM_INVALID, - "Input parameter[--dynamic_batch_size, --dynamic_image_size or --dynamic_dims]'s " + "[Check][Param] Input parameter[--dynamic_batch_size, --dynamic_image_size or --dynamic_dims]'s " "value size must be same, first group's size is %zu and another's is %zu.", shape_size, shape.size()); return PARAM_INVALID; @@ -506,7 +523,7 @@ Status CheckDynamicParams(const vector> &shapes) { for (auto dim : shape) { if (dim <= 0) { ErrorManager::GetInstance().ATCReportErrMessage("E10038", {"dim"}, {std::to_string(dim)}); - GELOGE(PARAM_INVALID, "Invalid dim %ld, all dims must be greater than 0", dim); + GELOGE(PARAM_INVALID, "[Check][Param] Invalid dim %ld, all dims must be greater than 0", dim); return PARAM_INVALID; } } @@ -514,8 +531,8 @@ Status CheckDynamicParams(const vector> &shapes) { } if (shapes_set.size() != shapes.size()) { ErrorManager::GetInstance().ATCReportErrMessage("E10039"); - GELOGE(PARAM_INVALID, - "Input parameter[--dynamic_batch_size, --dynamic_image_size or --dynamic_dims] exist duplicate shapes."); + GELOGE(PARAM_INVALID, "[Check][Param] Input parameter[--dynamic_batch_size, " + "--dynamic_image_size or --dynamic_dims] exist duplicate shapes."); return PARAM_INVALID; } return SUCCESS; @@ -537,9 +554,8 @@ Status CalcShape(const std::vector &batch_shape, GeShape &data_shape) { "E19012", {"function", "reason"}, {"CalcShape", "the batch shape count " + std::to_string(batch_shape.size()) + " does not match the data shape " + data_shape.ToString()}); - GELOGE(PARAM_INVALID, - "Failed to calc tensor shape, the batch shape count %zu, does not match the data shape %s", - batch_shape.size(), data_shape.ToString().c_str()); + GELOGE(PARAM_INVALID, "[Check][Param] Failed to calc tensor shape, the batch shape count %zu, " + "does not match the data shape %s", batch_shape.size(), data_shape.ToString().c_str()); return PARAM_INVALID; } data_shape.SetDim(i, batch_shape[batch_shape_index++]); @@ -550,8 +566,8 @@ Status CalcShape(const std::vector &batch_shape, GeShape &data_shape) { ErrorManager::GetInstance().ATCReportErrMessage( "E19012", {"function", "reason"}, {"CalcShape", "the batch shape count " + std::to_string(batch_shape.size()) + " does not match the data shape " + data_shape.ToString()}); - GELOGE(PARAM_INVALID, "Failed to calc tensor shape, the batch shape count %zu, does not match the data shape %s", - batch_shape.size(), data_shape.ToString().c_str()); + GELOGE(PARAM_INVALID, "[Check][Param] Failed to calc tensor shape, the batch shape count %zu, " + "does not match the data shape %s", batch_shape.size(), data_shape.ToString().c_str()); return PARAM_INVALID; } return SUCCESS; @@ -578,7 +594,8 @@ Status StampDynamicType(const OpDescPtr &op_desc) { if (!AttrUtils::SetInt(op_desc, ATTR_DYNAMIC_TYPE, dynamic_type)) { REPORT_CALL_ERROR("E19999", "Set Attr:%s to node:%s(%s) failed", ATTR_DYNAMIC_TYPE.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); - GELOGE(INTERNAL_ERROR, "Failed to add dynamic type attr for node %s", op_desc->GetName().c_str()); + GELOGE(INTERNAL_ERROR, "[Set][Attr] %s to node:%s(%s) failed", + ATTR_DYNAMIC_TYPE.c_str(), op_desc->GetName().c_str(), op_desc->GetType().c_str()); return INTERNAL_ERROR; } return SUCCESS; @@ -597,9 +614,8 @@ bool CheckDynamicBatchShape(const vector &shape, const string &data_nam if (shape[i] < 1) { ErrorManager::GetInstance().ATCReportErrMessage("E10018", {"index", "shape"}, {std::to_string(i), std::to_string(shape[i])}); - GELOGE(ge::PARAM_INVALID, - "Only batch N can be -1 when set --dynamic_batch_size, current data: %s shape[%zu] is %ld", - data_name.c_str(), i, shape[i]); + GELOGE(ge::PARAM_INVALID, "[Check][Param] Only batch N can be -1 when set --dynamic_batch_size, " + "current data: %s shape[%zu] is %ld", data_name.c_str(), i, shape[i]); return false; } } @@ -635,8 +651,8 @@ bool CheckDynamicImageSizeShape(const vector &shape, const string &data return true; } else { ErrorManager::GetInstance().ATCReportErrMessage("E10019"); - GELOGE(ge::PARAM_INVALID, - "--input_shape's shape is invalid, only height and width can be -1 when set --dynamic_image_size."); + GELOGE(ge::PARAM_INVALID, "[Check][Param] --input_shape's shape is invalid, only height and width can be -1 " + "when set --dynamic_image_size."); return false; } } diff --git a/ge/host_cpu_engine/CMakeLists.txt b/ge/host_cpu_engine/CMakeLists.txt index 8d84ee28..d54dacf7 100644 --- a/ge/host_cpu_engine/CMakeLists.txt +++ b/ge/host_cpu_engine/CMakeLists.txt @@ -1,10 +1,3 @@ -set(PROTO_LIST - "${METADEF_DIR}/proto/task.proto" -) - -protobuf_generate(ge PROTO_SRCS PROTO_HDRS ${PROTO_LIST}) -protobuf_generate(ge_atcstub PROTO_ATCSTUB_SRCS PROTO_ATCSTUB_HDRS ${PROTO_LIST}) - set(SRC_LIST "engine/host_cpu_engine.cc" "ops_kernel_store/host_cpu_ops_kernel_info.cc" @@ -17,7 +10,11 @@ set(CPU_OPS_KERNEL_LIST ) ############ libhost_cpu_engine.so ############ -add_library(host_cpu_engine SHARED ${SRC_LIST} ${PROTO_HDRS}) +add_library(host_cpu_engine SHARED ${SRC_LIST}) + +add_dependencies(host_cpu_engine + graphengine_protos +) target_compile_options(host_cpu_engine PRIVATE -Werror @@ -40,7 +37,7 @@ target_include_directories(host_cpu_engine PRIVATE ${METADEF_DIR}/inc/external ${METADEF_DIR}/inc/external/graph ${CMAKE_BINARY_DIR} - ${CMAKE_BINARY_DIR}/proto/ge + ${CMAKE_BINARY_DIR}/proto/graphengine_protos #### yellow zone #### ${GE_CODE_DIR}/../inc #### blue zone #### @@ -62,7 +59,11 @@ target_link_libraries(host_cpu_engine PRIVATE ) ############ atcstub/libhost_cpu_engine.so ############ -add_library(atc_host_cpu_engine SHARED ${SRC_LIST} ${PROTO_ATCSTUB_HDRS}) +add_library(atc_host_cpu_engine SHARED ${SRC_LIST}) + +add_dependencies(atc_host_cpu_engine + graphengine_protos +) target_compile_options(atc_host_cpu_engine PRIVATE -Werror @@ -85,7 +86,7 @@ target_include_directories(atc_host_cpu_engine PRIVATE ${METADEF_DIR}/inc/external ${METADEF_DIR}/inc/external/graph ${CMAKE_BINARY_DIR} - ${CMAKE_BINARY_DIR}/proto/ge_atcstub + ${CMAKE_BINARY_DIR}/proto/graphengine_protos #### yellow zone #### ${GE_CODE_DIR}/../inc #### blue zone #### @@ -114,6 +115,10 @@ set_target_properties(atc_host_cpu_engine PROPERTIES ############ libhost_cpu_opskernel_builder.so ############ add_library(host_cpu_opskernel_builder SHARED ${CPU_OPS_KERNEL_LIST}) +add_dependencies(host_cpu_opskernel_builder + graphengine_protos +) + target_compile_options(host_cpu_opskernel_builder PRIVATE -Werror -fno-common @@ -135,7 +140,7 @@ target_include_directories(host_cpu_opskernel_builder PRIVATE ${METADEF_DIR}/inc/external ${METADEF_DIR}/inc/external/graph ${CMAKE_BINARY_DIR} - ${CMAKE_BINARY_DIR}/proto/ge + ${CMAKE_BINARY_DIR}/proto/graphengine_protos #### yellow zone #### ${GE_CODE_DIR}/../inc #### blue zone #### @@ -160,6 +165,10 @@ target_link_libraries(host_cpu_opskernel_builder PRIVATE ############ atclib/libhost_cpu_opskernel_builder.so ############ add_library(atc_host_cpu_opskernel_builder SHARED ${CPU_OPS_KERNEL_LIST}) +add_dependencies(atc_host_cpu_opskernel_builder + graphengine_protos +) + target_compile_options(atc_host_cpu_opskernel_builder PRIVATE -Werror -fno-common @@ -181,7 +190,7 @@ target_include_directories(atc_host_cpu_opskernel_builder PRIVATE ${METADEF_DIR}/inc/external ${METADEF_DIR}/inc/external/graph ${CMAKE_BINARY_DIR} - ${CMAKE_BINARY_DIR}/proto/ge + ${CMAKE_BINARY_DIR}/proto/graphengine_protos #### yellow zone #### ${GE_CODE_DIR}/../inc #### blue zone #### @@ -211,6 +220,10 @@ set_target_properties(atc_host_cpu_opskernel_builder PROPERTIES ############ libhost_cpu_opskernel_builder.a ############ add_library(host_cpu_opskernel_builder_static STATIC ${CPU_OPS_KERNEL_LIST}) +add_dependencies(host_cpu_opskernel_builder_static + graphengine_protos +) + target_compile_options(host_cpu_opskernel_builder_static PRIVATE -Werror -fno-common @@ -233,7 +246,7 @@ target_include_directories(host_cpu_opskernel_builder_static PRIVATE ${METADEF_DIR}/inc/external ${METADEF_DIR}/inc/external/graph ${CMAKE_BINARY_DIR} - ${CMAKE_BINARY_DIR}/proto/ge + ${CMAKE_BINARY_DIR}/proto/graphengine_protos #### yellow zone #### ${GE_CODE_DIR}/../inc #### blue zone #### diff --git a/ge/hybrid/executor/hybrid_execution_context.h b/ge/hybrid/executor/hybrid_execution_context.h index f2628409..489d6d99 100644 --- a/ge/hybrid/executor/hybrid_execution_context.h +++ b/ge/hybrid/executor/hybrid_execution_context.h @@ -62,6 +62,7 @@ struct GraphExecutionContext { const HybridModel *model = nullptr; const GEThreadLocalContext *ge_context = nullptr; rtStream_t stream = nullptr; + rtStream_t hccl_stream = nullptr; rtContext_t rt_context = nullptr; rtContext_t rt_gen_context = nullptr; std::unique_ptr callback_manager = nullptr; diff --git a/ge/hybrid/executor/hybrid_model_executor.cc b/ge/hybrid/executor/hybrid_model_executor.cc index f8635a97..d8939175 100755 --- a/ge/hybrid/executor/hybrid_model_executor.cc +++ b/ge/hybrid/executor/hybrid_model_executor.cc @@ -50,7 +50,7 @@ Status HybridModelExecutor::Execute(HybridModelExecutor::ExecuteArgs &args) { auto root_graph_item = model_->GetRootGraphItem(); GE_CHECK_NOTNULL(root_graph_item); - if (root_graph_item->IsDynamic()) { + if (root_graph_item->IsDynamic() && !model_->IsSingleOp()) { GE_CHK_STATUS_RET(CheckInputShapeByShapeRange(root_graph_item, args), "[%s] check input node shape by shape range failed.", root_graph_item->GetName().c_str()); @@ -90,7 +90,7 @@ Status HybridModelExecutor::ExecuteGraphInternal(SubgraphExecutor &executor, int32_t device_id = static_cast(device_id_); auto &prof_mgr = ProfilingManager::Instance(); // tag_id 0 means step begin, 1 meas step end. - if (!model_->IsSingleOp() && prof_mgr.ProfilingModelLoadOn()) { + if (!model_->IsSingleOp()) { GE_CHK_STATUS_RET_NOLOG(prof_mgr.ProfileStepInfo(index_id, model_id, 0, stream_, device_id)); } @@ -98,7 +98,7 @@ Status HybridModelExecutor::ExecuteGraphInternal(SubgraphExecutor &executor, "Failed to execute partitioned call."); RECORD_MODEL_EXECUTION_EVENT(&context_, "[ExecuteAsync] End"); - if (!model_->IsSingleOp() && prof_mgr.ProfilingModelLoadOn()) { + if (!model_->IsSingleOp()) { GE_CHK_STATUS_RET_NOLOG(prof_mgr.ProfileStepInfo(index_id, model_id, 1, stream_, device_id)); } diff --git a/ge/hybrid/executor/hybrid_model_pipeline_executor.cc b/ge/hybrid/executor/hybrid_model_pipeline_executor.cc index ba24d78d..c0bd5c7d 100644 --- a/ge/hybrid/executor/hybrid_model_pipeline_executor.cc +++ b/ge/hybrid/executor/hybrid_model_pipeline_executor.cc @@ -18,14 +18,26 @@ const char *const kEnvProfilingLevel = "HYBRID_PROFILING_LEVEL"; StageExecutor::StageExecutor(int id, HybridModel *model, PipeExecutionConfig *config) : id_(id), model_(model), pipe_config_(config) {} -StageExecutor::~StageExecutor() { GELOGD("~StageExecutor(), id = %d", id_); } +StageExecutor::~StageExecutor() { + GELOGD("~StageExecutor(), id = %d", id_); + if (stream_ != nullptr) { + GE_CHK_RT(rtStreamDestroy(stream_)); + stream_ = nullptr; + } + if (hccl_stream_ != nullptr) { + GE_CHK_RT(rtStreamDestroy(hccl_stream_)); + hccl_stream_ = nullptr; + } +} Status StageExecutor::Init() { GELOGD("[Executor: %d] Start to init StateExecutor", id_); context_.rt_context = pipe_config_->rt_context; GE_CHK_STATUS_RET_NOLOG(InitExecutionContext()); GE_CHK_RT_RET(rtStreamCreate(&stream_, RT_STREAM_PRIORITY_DEFAULT)); + GE_CHK_RT_RET(rtStreamCreate(&hccl_stream_, RT_STREAM_PRIORITY_DEFAULT)); context_.stream = stream_; + context_.hccl_stream = hccl_stream_; root_graph_executor_.reset(new (std::nothrow) SubgraphExecutor(model_->GetRootGraphItem(), &context_)); GE_CHECK_NOTNULL(root_graph_executor_); @@ -78,11 +90,11 @@ Status StageExecutor::Start(const std::vector &inputs, const std::v if (task_info.event != nullptr) { GELOGD("[%d] Add StreamWaitEvent", id_); GE_CHK_RT_RET(rtStreamWaitEvent(stream_, task_info.event)); - RECORD_MODEL_EXECUTION_EVENT(&context_, "[iteration = %ld] [Stage = %d] End", task_info.iteration - 1, + RECORD_MODEL_EXECUTION_EVENT(&context_, "[iteration = %ld] [Stage = %d] EventWait End", task_info.iteration, task_info.stage); } - RECORD_MODEL_EXECUTION_EVENT(&context_, "[iteration = %lld] [Stage = %d] Start", task_info.iteration, + RECORD_MODEL_EXECUTION_EVENT(&context_, "[iteration = %ld] [Stage = %d] Start", task_info.iteration, task_info.stage); if (task_info.stage == 0) { @@ -102,6 +114,10 @@ Status StageExecutor::Start(const std::vector &inputs, const std::v StageTask next_task; next_task.stage = task_info.stage; next_task.iteration = task_info.iteration + 1; + if ((task_info.iteration + 1) % iteration_count > 0) { + GE_CHK_RT_RET(rtEventCreate(&next_task.event)); + GE_CHK_RT_RET(rtEventRecord(next_task.event, context_.hccl_stream)); + } auto sync_result = Synchronize(); if (sync_result != SUCCESS) { @@ -110,15 +126,22 @@ Status StageExecutor::Start(const std::vector &inputs, const std::v id_, sync_result, task_info.iteration); REPORT_CALL_ERROR("E19999", "[Executor: %d] Failed to sync result:%d. iteration = %ld", id_, sync_result, task_info.iteration); - context_.profiler->Dump(std::cout); + if (context_.profiler != nullptr) { + context_.profiler->Dump(std::cout); + } context_.callback_manager->Destroy(); RuntimeInferenceContext::DestroyContext(std::to_string(context_.context_id)); return sync_result; } + if (task_info.event != nullptr) { + GE_CHK_RT_RET(rtEventDestroy(task_info.event)); + RECORD_MODEL_EXECUTION_EVENT(&context_, "[iteration = %ld] [Stage = %d] EventDestroy End", task_info.iteration, + task_info.stage); + } RECORD_MODEL_EXECUTION_EVENT(&context_, "[iteration = %ld] [Stage = %d] End", task_info.iteration, task_info.stage); - // if not end stage + // if end stage if (task_info.stage >= pipe_config_->num_stages - 1) { RECORD_MODEL_EXECUTION_EVENT(&context_, "[iteration = %ld] Schedule End", task_info.iteration); GELOGD("[Executor: %d] End of iteration [%ld]", id_, task_info.iteration); @@ -163,6 +186,7 @@ Status StageExecutor::InitExecutionContext() { context_.callback_manager = std::unique_ptr(new (std::nothrow) CallbackManager()); GE_CHECK_NOTNULL(context_.callback_manager); context_.dump_properties = DumpManager::GetInstance().GetDumpProperties(context_.session_id); + context_.is_eos_ = false; if (IsLogEnable(GE_MODULE_NAME, DLOG_DEBUG)) { context_.trace_enabled = true; } diff --git a/ge/hybrid/executor/hybrid_model_pipeline_executor.h b/ge/hybrid/executor/hybrid_model_pipeline_executor.h index cb08d872..c59e1462 100644 --- a/ge/hybrid/executor/hybrid_model_pipeline_executor.h +++ b/ge/hybrid/executor/hybrid_model_pipeline_executor.h @@ -63,6 +63,7 @@ class StageExecutor { StageExecutor *next_executor_ = nullptr; rtStream_t stream_ = nullptr; + rtStream_t hccl_stream_ = nullptr; }; class HybridModelPipelineExecutor { diff --git a/ge/hybrid/executor/node_done_manager.cc b/ge/hybrid/executor/node_done_manager.cc index d31765c2..0ea04661 100644 --- a/ge/hybrid/executor/node_done_manager.cc +++ b/ge/hybrid/executor/node_done_manager.cc @@ -121,5 +121,10 @@ void NodeDoneManager::Reset(const NodePtr &node) { GELOGD("[%s] Node reset.", node->GetName().c_str()); } } + +void NodeDoneManager::Reset() { + subjects_.clear(); + destroyed_ = false; +} } // namespace hybrid } // namespace ge diff --git a/ge/hybrid/executor/node_done_manager.h b/ge/hybrid/executor/node_done_manager.h index 292d1369..bedbff3d 100644 --- a/ge/hybrid/executor/node_done_manager.h +++ b/ge/hybrid/executor/node_done_manager.h @@ -35,6 +35,8 @@ class NodeDoneManager { void Destroy(); + void Reset(); + private: class Cond { public: diff --git a/ge/hybrid/executor/node_state.cc b/ge/hybrid/executor/node_state.cc index aaa7801f..fd47cfb2 100644 --- a/ge/hybrid/executor/node_state.cc +++ b/ge/hybrid/executor/node_state.cc @@ -104,11 +104,47 @@ void ShapeInferenceState::UpdateInputShapeFuture(int idx, ShapeFuture &&future) } } +Status ShapeInferenceState::UpdateInputForMerge(const GraphExecutionContext &context) { + int merge_index = -1; + const auto &guard = node_item.MutexGuard("UpdateInputForMerge"); + if (!AttrUtils::GetInt(node_item.op_desc, ATTR_NAME_MERGE_INPUT_INDEX, merge_index)) { + GELOGE(FAILED, "[%s] Get attr %s failed", node_item.NodeName().c_str(), ATTR_NAME_MERGE_INPUT_INDEX.c_str()); + return FAILED; + } + + if (merge_index < 0 || static_cast(merge_index) >= input_tensor_desc.size()) { + GELOGE(FAILED, "[%s] merge index: %d invalid, should in range[0, %zu)", + node_item.NodeName().c_str(), merge_index, input_tensor_desc.size()); + return FAILED; + } + + auto dst_tensor_desc = node_item.MutableInputDesc(merge_index); + GE_CHECK_NOTNULL(dst_tensor_desc); + + int64_t tensor_size = -1; + auto &tensor_desc = input_tensor_desc[merge_index]; + (void)TensorUtils::GetSize(tensor_desc, tensor_size); + + dst_tensor_desc->SetShape(tensor_desc.MutableShape()); + dst_tensor_desc->SetOriginShape(tensor_desc.GetOriginShape()); + (void)TensorUtils::SetSize(*dst_tensor_desc, tensor_size); + (void)guard; + GELOGD("[%s] Update input shape [%u] with shape: [%s] and ori_shape: [%s], tensor size = %ld", + node_item.NodeName().c_str(), merge_index, dst_tensor_desc->GetShape().ToString().c_str(), + dst_tensor_desc->GetOriginShape().ToString().c_str(), tensor_size); + + return SUCCESS; +} + Status ShapeInferenceState::AwaitShapesReady(const GraphExecutionContext &context) { if (!node_item.is_dynamic) { return SUCCESS; } std::unique_lock lk(mu_); + if (node_item.IsMergeOp()) { + return UpdateInputForMerge(context); + } + if (num_pending_shapes_ > 0) { GELOGD("[%s] Await pending shape or shape future start.", node_item.NodeName().c_str()); int try_count = 0; @@ -169,7 +205,7 @@ Status ShapeInferenceState::AwaitShapesReady(const GraphExecutionContext &contex int64_t tensor_size = -1; (void) TensorUtils::GetSize(*src_tensor_desc, tensor_size); - GELOGD("[%s] Update input shape [%u] with shape: [%s] and ori_shape: [%s], index = %zu", + GELOGD("[%s] Update input shape [%u] with shape: [%s] and ori_shape: [%s], tensor size = %ld", node_item.NodeName().c_str(), idx, src_tensor_desc->GetShape().ToString().c_str(), @@ -270,31 +306,15 @@ std::shared_ptr NodeState::GetTaskContext() { return task_context_; } -void NodeState::ResetContext(int group) { - SetGroup(group); - if (loop_count_ == 0) { - ++loop_count_; - return; - } - - ++loop_count_; - if (loop_count_ == UINT64_MAX) { - loop_count_ = 1; - } +void NodeState::ResetContext(uint64_t loop_count) { + loop_count_ = loop_count; switch_index_ = -1; - const auto &guard = node_item_->MutexGuard("ResetContext"); - shape_inference_state_.InitShapeState(); subgraph_context_->ResetContext(node_item_->node); - GELOGD("Node[%s] in while loop, current loop: %lu, merge index: %d", GetName().c_str(), loop_count_, merge_index_); - (void)guard; -} - -void NodeState::ResetSchedule() { - std::lock_guard lk(mu_); data_scheduled_ = static_cast(node_item_->root_data_.size()); ctrl_scheduled_ = static_cast(node_item_->root_ctrl_.size()); - GELOGD("[%s] set schedule for root nodes, data: %u, ctrl: %u", GetName().c_str(), data_scheduled_, ctrl_scheduled_); + GELOGD("[%s] in while loop, loop count: %lu, data scheduled: %u, ctrl scheduled: %u, merge index: %d", + GetName().c_str(), loop_count_, data_scheduled_, ctrl_scheduled_, merge_index_); } Status NodeState::NodeScheduled(const std::function &ready) const { @@ -302,14 +322,14 @@ Status NodeState::NodeScheduled(const std::function &rea for (const auto &node : node_item_->data_send_) { const auto &dst_node_state = subgraph_context_->GetOrCreateNodeState(node); GE_CHECK_NOTNULL(dst_node_state); - dst_node_state->SetDataSchedule(node_item_, ready); + dst_node_state->SetDataSchedule(*this, ready); } // Schedule ctrl output. for (const auto &node : node_item_->ctrl_send_) { const auto &dst_node_state = subgraph_context_->GetOrCreateNodeState(node); GE_CHECK_NOTNULL(dst_node_state); - dst_node_state->SetCtrlSchedule(node_item_, ready); + dst_node_state->SetCtrlSchedule(*this, ready); } // Schedule switch group. @@ -318,7 +338,7 @@ Status NodeState::NodeScheduled(const std::function &rea for (const auto &node : node_item_->switch_groups_[switch_index_]) { const auto &dst_node_state = subgraph_context_->GetOrCreateNodeState(node); GE_CHECK_NOTNULL(dst_node_state); - dst_node_state->SetCtrlSchedule(node_item_, ready); + dst_node_state->SetCtrlSchedule(*this, ready); } } @@ -326,36 +346,45 @@ Status NodeState::NodeScheduled(const std::function &rea } bool NodeState::IsScheduleReady() const { - GELOGD("[%s] data[input: %zu, scheduled: %u], ctrl[input: %zu, scheduled: %u]", GetName().c_str(), - node_item_->data_recv_.size(), data_scheduled_, node_item_->ctrl_recv_.size(), ctrl_scheduled_); - if (ctrl_scheduled_ != node_item_->ctrl_recv_.size()) { - return false; - } - + GELOGD("[%s] loop[%lu] data[input: %zu, scheduled: %u], ctrl[input: %zu+%zu, scheduled: %u]", + GetName().c_str(), loop_count_, node_item_->data_recv_.size(), data_scheduled_, + node_item_->ctrl_recv_.size(), node_item_->GetMergeCtrl(loop_count_ == 0 ? 0 : 1), ctrl_scheduled_); if (node_item_->IsMergeOp()) { + if (ctrl_scheduled_ != node_item_->GetMergeCtrl(loop_count_ == 0 ? 0 : 1) + node_item_->ctrl_recv_.size()) { + return false; + } + return data_scheduled_ > 0; } + if (ctrl_scheduled_ != node_item_->ctrl_recv_.size()) { + return false; + } + // Exit may feed loop times... return data_scheduled_ >= node_item_->data_recv_.size(); } -void NodeState::SetDataSchedule(const NodeItem *node_item, const std::function &ready) { - GELOGD("[%s] data schedule node[%s], data num: %zu, current scheduled: %u, ctrl num: %zu, current scheduled: %u", - node_item->node_name.c_str(), GetName().c_str(), node_item_->data_recv_.size(), data_scheduled_, - node_item_->ctrl_recv_.size(), ctrl_scheduled_); +void NodeState::SetDataSchedule(const NodeState &node_state, const std::function &ready) { + GELOGD("[%s] schedule [%s], loop[%lu -> %lu], data[num: %zu, scheduled: %u], ctrl[num: %zu+%zu, scheduled: %u]", + node_state.GetName().c_str(), GetName().c_str(), loop_count_, node_state.loop_count_, + node_item_->data_recv_.size(), data_scheduled_, node_item_->ctrl_recv_.size(), + node_item_->GetMergeCtrl(loop_count_ == 0 ? 0 : 1), ctrl_scheduled_); std::lock_guard lk(mu_); + if (loop_count_ != node_state.loop_count_) { + ResetContext(node_state.loop_count_); + } ++data_scheduled_; if (node_item_->IsMergeOp()) { - const auto it = node_item_->data_recv_.find(node_item); + const auto it = node_item_->data_recv_.find(node_state.node_item_); if (it != node_item_->data_recv_.end()) { merge_index_ = it->second; (void)AttrUtils::SetInt(node_item_->node->GetOpDesc(), ATTR_NAME_MERGE_INPUT_INDEX, it->second); - GELOGD("[%s] scheduled, [%s] set merge index: %d", node_item->node_name.c_str(), GetName().c_str(), it->second); + GELOGD("[%s] scheduled, [%s] set merge index: %d", node_state.GetName().c_str(), GetName().c_str(), it->second); } else { - GELOGW("[%s] scheduled, [%s] not followed", node_item->node_name.c_str(), GetName().c_str()); + GELOGW("[%s] scheduled, [%s] not followed", node_state.GetName().c_str(), GetName().c_str()); } } @@ -364,12 +393,16 @@ void NodeState::SetDataSchedule(const NodeItem *node_item, const std::function &ready) { - GELOGD("[%s] ctrl schedule node[%s], data num: %zu, current scheduled: %u, ctrl num: %zu, current scheduled: %u", - node_item->node_name.c_str(), GetName().c_str(), node_item_->data_recv_.size(), data_scheduled_, - node_item_->ctrl_recv_.size(), ctrl_scheduled_); +void NodeState::SetCtrlSchedule(const NodeState &node_state, const std::function &ready) { + GELOGD("[%s] schedule [%s], loop[%lu -> %lu], data[num: %zu, scheduled: %u], ctrl[num: %zu+%zu, scheduled: %u]", + node_state.GetName().c_str(), GetName().c_str(), loop_count_, node_state.loop_count_, + node_item_->data_recv_.size(), data_scheduled_, node_item_->ctrl_recv_.size(), + node_item_->GetMergeCtrl(loop_count_ == 0 ? 0 : 1), ctrl_scheduled_); std::lock_guard lk(mu_); + if (loop_count_ != node_state.loop_count_) { + ResetContext(node_state.loop_count_); + } ++ctrl_scheduled_; if (IsScheduleReady()) { @@ -377,6 +410,23 @@ void NodeState::SetCtrlSchedule(const NodeItem *node_item, const std::function lk(mu_); + ++loop_count_; + if (loop_count_ == UINT64_MAX) { + loop_count_ = 1; + } + + ResetContext(loop_count_); +} + +void NodeState::RunLoopExit() { + GELOGD("Node[%s] run in loop, current count: %lu", GetName().c_str(), loop_count_); + std::lock_guard lk(mu_); + loop_count_ = 0; +} + void NodeState::SetScheduleFuture(std::future &&future) { schedule_future_ = std::move(future); } diff --git a/ge/hybrid/executor/node_state.h b/ge/hybrid/executor/node_state.h index 49861611..e4afdb9f 100644 --- a/ge/hybrid/executor/node_state.h +++ b/ge/hybrid/executor/node_state.h @@ -67,6 +67,8 @@ struct ShapeInferenceState { const NodeItem &node_item; private: + Status UpdateInputForMerge(const GraphExecutionContext &context); + friend struct NodeState; std::vector> shape_futures; // do not directly update op_desc, in case race condition across pipelines @@ -110,9 +112,8 @@ struct NodeState { return node_item_->IsControlFlowOp() || node_item_->shape_inference_type >= DEPEND_SHAPE_RANGE; } - void ResetContext(int group); - - void ResetSchedule(); + void RunLoopNext(); + void RunLoopExit(); Status NodeScheduled(const std::function &ready) const; @@ -164,8 +165,9 @@ struct NodeState { private: bool IsScheduleReady() const; - void SetDataSchedule(const NodeItem *node_item, const std::function &ready); - void SetCtrlSchedule(const NodeItem *node_item, const std::function &ready); + void SetDataSchedule(const NodeState &node_state, const std::function &ready); + void SetCtrlSchedule(const NodeState &node_state, const std::function &ready); + void ResetContext(uint64_t loop_count); const NodeItem *node_item_ = nullptr; std::shared_ptr kernel_task_ = nullptr; diff --git a/ge/hybrid/executor/subgraph_context.cc b/ge/hybrid/executor/subgraph_context.cc index 9a9a97c2..afd8ca79 100644 --- a/ge/hybrid/executor/subgraph_context.cc +++ b/ge/hybrid/executor/subgraph_context.cc @@ -15,8 +15,6 @@ */ #include "subgraph_context.h" - -#include "common/debug/log.h" #include "hybrid/executor/hybrid_model_executor.h" namespace ge { @@ -25,6 +23,13 @@ SubgraphContext::SubgraphContext(const GraphItem *graph_item, const GraphExecuti : graph_item_(graph_item), execution_context_(execution_context) { } +SubgraphContext::~SubgraphContext() { + if (mmRWLockDestroy(&rw_lock_) != EN_OK) { + REPORT_CALL_ERROR("E19999", "Destroy rw_lock failed"); + GELOGE(INTERNAL_ERROR, "[RWLock][Destroy] Destroy rw_lock failed"); + } +} + Status SubgraphContext::Init() { GE_CHECK_NOTNULL(graph_item_); GELOGD("[%s] Start to init subgraph context. total inputs = %d, total outputs = %d", @@ -33,22 +38,66 @@ Status SubgraphContext::Init() { graph_item_->TotalOutputs()); all_inputs_.resize(static_cast(graph_item_->TotalInputs())); all_outputs_.resize(static_cast(graph_item_->TotalOutputs())); - + if (mmRWLockInit(&rw_lock_) != EN_OK) { + REPORT_CALL_ERROR("E19999", "Init rw_lock failed"); + GELOGE(INTERNAL_ERROR, "[RWLock][Init] Init rw_lock failed"); + return INTERNAL_ERROR; + } return SUCCESS; } +void SubgraphContext::SetGroup(int group) { + group_ = group; +} + void SubgraphContext::ResetContext(const NodePtr &node) { node_done_manager_.Reset(node); } NodeStatePtr SubgraphContext::GetOrCreateNodeState(const NodeItem *node_item) { - std::lock_guard lk(mu_); + GELOGD("[%s] lock for read", node_item->NodeName().c_str()); + if (mmRWLockRDLock(&rw_lock_) != EN_OK) { + REPORT_CALL_ERROR("E19999", "[Node:%s] Lock for read failed", node_item->NodeName().c_str()); + GELOGE(INTERNAL_ERROR, "[RWLock][Lock][Node:%s] Lock for read failed", node_item->NodeName().c_str()); + return nullptr; + } + const auto &iter = node_states_.find(node_item); + if (iter != node_states_.end()) { + auto state = iter->second; + GELOGD("[%s] unlock for read", node_item->NodeName().c_str()); + if (mmRDLockUnLock(&rw_lock_) != EN_OK) { + REPORT_CALL_ERROR("E19999", "[Node:%s] Unlock for read failed", node_item->NodeName().c_str()); + GELOGE(INTERNAL_ERROR, "[RWLock][Unlock][Node:%s] Unlock for read failed", node_item->NodeName().c_str()); + return nullptr; + } + return state; + } + GELOGD("[%s] unlock for read", node_item->NodeName().c_str()); + if (mmRDLockUnLock(&rw_lock_) != EN_OK) { + REPORT_CALL_ERROR("E19999", "[Node:%s] Unlock for read failed", node_item->NodeName().c_str()); + GELOGE(INTERNAL_ERROR, "[RWLock][Unlock][Node:%s] Unlock for read failed", node_item->NodeName().c_str()); + return nullptr; + } + + GELOGD("[%s] lock for write", node_item->NodeName().c_str()); + if (mmRWLockWRLock(&rw_lock_) != EN_OK) { + REPORT_CALL_ERROR("E19999", "[Node:%s] Lock for write failed", node_item->NodeName().c_str()); + GELOGE(INTERNAL_ERROR, "[RWLock][Lock][Node:%s] Lock for write failed", node_item->NodeName().c_str()); + return nullptr; + } auto &node_state = node_states_[node_item]; if (node_state == nullptr) { const auto &guard = node_item->MutexGuard("GetOrCreateNodeState"); node_state.reset(new(std::nothrow)NodeState(*node_item, this)); + node_state->SetGroup(group_); (void)guard; } + GELOGD("[%s] unlock for write", node_item->NodeName().c_str()); + if (mmWRLockUnLock(&rw_lock_) != EN_OK) { + REPORT_CALL_ERROR("E19999", "[Node:%s] Unlock for write failed", node_item->NodeName().c_str()); + GELOGE(INTERNAL_ERROR, "[RWLock][Unlock][Node:%s] Unlock for write failed", node_item->NodeName().c_str()); + return nullptr; + } return node_state; } @@ -144,5 +193,13 @@ void SubgraphContext::OnError(Status error) { void SubgraphContext::NodeDone(const NodePtr &node) { node_done_manager_.NodeDone(node); } + +void SubgraphContext::Reset() { + node_done_manager_.Reset(); + if (mmRWLockWRLock(&rw_lock_) == EN_OK) { + node_states_.clear(); + (void)mmWRLockUnLock(&rw_lock_); + } +} } // namespace hybrid } // namespace ge diff --git a/ge/hybrid/executor/subgraph_context.h b/ge/hybrid/executor/subgraph_context.h index ff692ed9..303382c1 100644 --- a/ge/hybrid/executor/subgraph_context.h +++ b/ge/hybrid/executor/subgraph_context.h @@ -18,7 +18,7 @@ #define GE_HYBRID_EXECUTOR_ITERATION_CONTEXT_H_ #include - +#include "mmpa/mmpa_api.h" #include "hybrid/common/tensor_value.h" #include "hybrid/executor/hybrid_execution_context.h" #include "hybrid/executor/node_state.h" @@ -31,10 +31,12 @@ namespace hybrid { class SubgraphContext { public: explicit SubgraphContext(const GraphItem *graph_item, const GraphExecutionContext *execution_context); - ~SubgraphContext() = default; + ~SubgraphContext(); Status Init(); + void SetGroup(int group); void ResetContext(const NodePtr &node); + void Reset(); NodeStatePtr GetOrCreateNodeState(const NodeItem *node_item); void OnError(Status error); @@ -52,11 +54,12 @@ class SubgraphContext { friend class TaskContext; const GraphItem *graph_item_; const GraphExecutionContext *execution_context_; - std::mutex mu_; + mmRWLock_t rw_lock_; std::vector all_inputs_; std::vector all_outputs_; NodeDoneManager node_done_manager_; std::unordered_map node_states_; + int group_ = -1; }; } // namespace hybrid } // namespace ge diff --git a/ge/hybrid/executor/subgraph_executor.cc b/ge/hybrid/executor/subgraph_executor.cc index 60895c7e..612e7565 100644 --- a/ge/hybrid/executor/subgraph_executor.cc +++ b/ge/hybrid/executor/subgraph_executor.cc @@ -242,7 +242,6 @@ Status SubgraphExecutor::PrepareNode(const NodeItem &node_item, int group) { auto node_state = subgraph_context_->GetOrCreateNodeState(&node_item); GE_CHECK_NOTNULL(node_state); - node_state->ResetContext(group); auto p_node_state = node_state.get(); if (node_item.node_type == NETOUTPUT) { @@ -367,7 +366,6 @@ Status SubgraphExecutor::NodeScheduled(NodeState *node_state) { }; GE_CHK_STATUS_RET_NOLOG(node_state->NodeScheduled(callback)); - node_state->ResetSchedule(); RECORD_CALLBACK_EVENT(context_, node_state->GetName().c_str(), "[NodeScheduled] End"); return SUCCESS; }); @@ -539,6 +537,7 @@ Status SubgraphExecutor::LaunchTasks() { Status SubgraphExecutor::ScheduleTasks(int group) { GELOGD("[%s] Start to schedule prepare workers.", graph_item_->GetName().c_str()); + subgraph_context_->SetGroup(group); auto prepare_future = std::async(std::launch::async, [&]() -> Status { GetContext().SetSessionId(context_->session_id); GetContext().SetContextId(context_->context_id); @@ -704,7 +703,21 @@ Status SubgraphExecutor::PartialExecuteAsync(int task_group) { Status SubgraphExecutor::InitForPartialExecution(const vector &inputs, const vector &input_desc) { - return Init(inputs, input_desc); + if (subgraph_context_ == nullptr) { + return Init(inputs, input_desc); + } + subgraph_context_->Reset(); + if (graph_item_->IsDynamic()) { + GE_CHK_STATUS_RET(InitInputsForUnknownShape(inputs, input_desc), + "[%s] Failed to set inputs.", + graph_item_->GetName().c_str()); + } else { + GE_CHK_STATUS_RET(InitInputsForKnownShape(inputs), + "[Invoke][InitInputsForKnownShape][%s] Failed to init subgraph executor for known shape subgraph", + graph_item_->GetName().c_str()); + } + + return SUCCESS; } } // namespace hybrid } // namespace ge diff --git a/ge/hybrid/model/hybrid_model_builder.cc b/ge/hybrid/model/hybrid_model_builder.cc index 86530fe0..b00b8ec8 100755 --- a/ge/hybrid/model/hybrid_model_builder.cc +++ b/ge/hybrid/model/hybrid_model_builder.cc @@ -21,6 +21,7 @@ #include "graph/ge_context.h" #include "graph/build/memory/var_mem_assign_util.h" #include "graph/debug/ge_attr_define.h" +#include "graph/common/omg_util.h" #include "graph/load/model_manager/model_utils.h" #include "graph/load/model_manager/model_manager.h" #include "graph/manager/graph_var_manager.h" @@ -43,8 +44,9 @@ const uint64_t kProfilingBpEndLogid = 2U; const uint64_t kProfilingIterEndLogid = 65535U; const int kBytes = 8; const int kDecimal = 10; -const uint8_t kStreamActiveIdx = 0; -const uint8_t kStreamActiveNum = 1; +const uint8_t kLoopEnterIdx = 0; +const uint8_t kLoopIterationIdx = 1; +const uint8_t kLoopMergeSize = 2; const uint8_t kStreamSwitchIdx = 1; const uint8_t kStreamSwitchNum = 2; const uint32_t kStringHeadElems = 2; @@ -57,6 +59,10 @@ const char *const kProfilingArNode = "ProfilingAllReduceNode"; const char *const kEngineNameRts = "DNN_VM_RTS_OP_STORE"; const char *const kForceInfershape = "_force_infershape_when_running"; +const std::set kExecutionDependentTypes{ IF, STATELESSIF, CASE, STREAMSWITCH }; +const std::set kMergeInputSkipTypes{ STREAMACTIVE, STREAMSWITCH, CONSTANT, CONSTANTOP }; +const std::set kStreamActiveTypes{ ENTER, REFENTER, NEXTITERATION, REFNEXTITERATION }; + Status SetOutputNameAttr(ComputeGraph &graph) { vector output_names; for (const auto &node : graph.GetDirectNode()) { @@ -389,7 +395,7 @@ Status HybridModelBuilder::ParseDependentInputNodes(NodeItem &node_item, const s } // cond or branch need to be prepared before the execution of IF or CASE - if (node_item.node_type == IF || node_item.node_type == STATELESSIF || node_item.node_type == CASE) { + if (kExecutionDependentTypes.count(node_item.node_type) > 0) { auto src_node = NodeUtils::GetInDataNodeByIndex(*ge_node, 0); // cond input GE_CHECK_NOTNULL(src_node); auto src_node_item = MutableNodeItem(src_node); @@ -575,7 +581,7 @@ Status HybridModelBuilder::MergeInputNodes(ComputeGraph &graph) { auto in_nodes = root_node->GetInAllNodes(); std::set in_node_set(in_nodes.begin(), in_nodes.end()); for (auto &in_control_node : wrapped_node->GetInControlNodes()) { - if (in_node_set.count(in_control_node) == 0) { + if (in_node_set.count(in_control_node) == 0 && kMergeInputSkipTypes.count(root_node->GetType()) == 0) { GELOGD("[%s] Restore control edge to [%s]", in_control_node->GetName().c_str(), root_node->GetName().c_str()); GE_CHECK_NOTNULL(in_control_node->GetOutControlAnchor()); (void) in_control_node->GetOutControlAnchor()->LinkTo(root_node->GetInControlAnchor()); @@ -1005,14 +1011,18 @@ Status HybridModelBuilder::InitConstantOps() { // Tensors return by api GetWeights share data with proto, whose addr is not confirmed to be aligned GeTensor aligned_tensor = ge_tensor->Clone(); GELOGD("Init tensor with host constant %s size = %zu", var_name.c_str(), aligned_tensor.MutableData().GetSize()); - if (MemManager::Instance().HostMemInstance(RT_MEMORY_HBM).Malloc(aligned_tensor.GetAlignedPtr(), - aligned_tensor.GetData().size()) == nullptr) { - GELOGE(MEMALLOC_FAILED, "[Malloc][HostMemory] for an existed GeTensor failed, model_name_:%s.", - GetGraphName()); - return MEMALLOC_FAILED; + if (aligned_tensor.GetData().size() > 0) { + if (MemManager::Instance().HostMemInstance(RT_MEMORY_HBM).Malloc(aligned_tensor.GetAlignedPtr(), + aligned_tensor.GetData().size()) == nullptr) { + GELOGE(MEMALLOC_FAILED, "[Malloc][HostMemory] for an existed GeTensor failed, model_name_:%s.", + GetGraphName()); + return MEMALLOC_FAILED; + } + var_tensor.reset(new(std::nothrow)TensorValue(aligned_tensor.MutableData().data(), + aligned_tensor.GetData().size())); + } else { + var_tensor.reset(new(std::nothrow)TensorValue(nullptr, 0)); } - var_tensor.reset(new(std::nothrow)TensorValue(aligned_tensor.MutableData().data(), - aligned_tensor.GetData().size())); } else { GE_CHK_STATUS_RET_NOLOG(VarNodeToTensor(var_node, var_tensor)); GELOGD("Init const op tensor. name = %s, size = %ld", var_name.c_str(), var_tensor->GetSize()); @@ -2278,8 +2288,6 @@ Status HybridModelBuilder::RelinkNextIteration() { } } - stream_merge_op_nodes_.clear(); - next_iteration_op_nodes_.clear(); return SUCCESS; } @@ -2367,10 +2375,12 @@ Status HybridModelBuilder::BuildControlFlowGroup(GraphItem &graph_item, const No } Status HybridModelBuilder::CreateNormalNodeGroup(const NodePtr &node, NodeItem *node_item) { - const auto out_ctrl_anchor = node->GetOutControlAnchor(); - for (const auto &peer_in_anchor : out_ctrl_anchor->GetPeerInControlAnchors()) { - const auto &dst_node = peer_in_anchor->GetOwnerNode(); + for (const auto &dst_node : node->GetOutControlNodes()) { GE_CHECK_NOTNULL(dst_node); + if ((dst_node->GetType() == STREAMACTIVE) && (kStreamActiveTypes.count(node->GetType()) == 0)) { + GELOGI("[%s] ignore control to [%s]", node->GetName().c_str(), dst_node->GetName().c_str()); + continue; + } NodeItem *dst_node_item = nullptr; GE_CHK_STATUS_RET(GetOrCreateNodeItem(dst_node, &dst_node_item), @@ -2380,27 +2390,84 @@ Status HybridModelBuilder::CreateNormalNodeGroup(const NodePtr &node, NodeItem * return SUCCESS; } +Status HybridModelBuilder::CreateMergeEnterGroup(const NodePtr &node, NodeItem *node_item) { + // Enter --> StreamActive --> StreamMerge + for (const auto &dst_node : node->GetOutControlNodes()) { + GE_CHECK_NOTNULL(dst_node); + if (dst_node->GetType() != STREAMMERGE) { + GELOGI("[%s] Skip Not StreamMerge node [%s]", node->GetName().c_str(), dst_node->GetName().c_str()); + continue; + } + NodeItem *dst_node_item = nullptr; + GE_CHK_STATUS_RET(GetOrCreateNodeItem(dst_node, &dst_node_item), + "[%s] failed to get or create node item", dst_node->GetName().c_str()); + // Set Enter Control to StreamMerge as Group 0. + dst_node_item->switch_groups_.resize(kLoopMergeSize); + dst_node_item->SetMergeCtrl(node_item, kLoopEnterIdx); + } + return SUCCESS; +} + +Status HybridModelBuilder::CreateMergeIterationGroup(const NodePtr &node, NodeItem *node_item) { + // NextIteration --> StreamActive {-->} StreamMerge + std::string node_name; + for (const auto &src_node : node->GetInControlNodes()) { + GE_CHECK_NOTNULL(src_node); + if (kNextIterationOpTypes.count(src_node->GetType()) == 0) { + GELOGI("[%s] Skip Not NextIteration node [%s]", node->GetName().c_str(), src_node->GetName().c_str()); + continue; + } + + if (!AttrUtils::GetStr(src_node->GetOpDesc(), ATTR_NAME_NEXT_ITERATION, node_name)) { + GELOGE(INTERNAL_ERROR, "[%s] input node [%s] expect attribute[%s] not found", + node->GetName().c_str(), src_node->GetName().c_str(), ATTR_NAME_NEXT_ITERATION.c_str()); + return INTERNAL_ERROR; + } + + const auto it = stream_merge_op_nodes_.find(node_name); + if (it == stream_merge_op_nodes_.end()) { + GELOGE(INTERNAL_ERROR, "[%s] expect StreamMerge[%s] not found", node->GetName().c_str(), node_name.c_str()); + return INTERNAL_ERROR; + } + + const auto &dst_node = it->second; + GE_CHECK_NOTNULL(dst_node); + NodeItem *dst_node_item = nullptr; + GE_CHK_STATUS_RET(GetOrCreateNodeItem(dst_node, &dst_node_item), "[%s] failed to get or create node item", + dst_node->GetName().c_str()); + // Set NextIteration Control to StreamMerge as Group 1. + dst_node_item->SetMergeCtrl(node_item, kLoopIterationIdx); + } + return SUCCESS; +} + Status HybridModelBuilder::CreateStreamActiveGroup(const NodePtr &node, NodeItem *node_item) { if (node_item->node_type != STREAMACTIVE) { GELOGE(INTERNAL_ERROR, "Called by %s is invalid", node_item->node_type.c_str()); return INTERNAL_ERROR; } - node_item->switch_groups_.resize(kStreamActiveNum); - const auto &out_ctrl_anchor = node->GetOutControlAnchor(); - for (const auto &peer_in_anchor : out_ctrl_anchor->GetPeerInControlAnchors()) { - const auto &dst_node = peer_in_anchor->GetOwnerNode(); - GE_CHECK_NOTNULL(dst_node); - if (dst_node->GetType() == STREAMMERGE) { - GELOGI("[%s] skip control node: %s", node->GetName().c_str(), dst_node->GetName().c_str()); - continue; - } + const auto ctrl_nodes = node->GetInControlNodes(); + if (ctrl_nodes.empty()) { + GELOGW("Skip no in control node: %s", node->GetName().c_str()); + return SUCCESS; + } - NodeItem *dst_node_item = nullptr; - GE_CHK_STATUS_RET(GetOrCreateNodeItem(dst_node, &dst_node_item), - "[%s] failed to get or create node item", dst_node->GetName().c_str()); - node_item->SetCtrlSend(dst_node_item, kStreamActiveIdx); + const auto IsEnterNode = [](const NodePtr &n) { + return kEnterOpTypes.count(n->GetType()) > 0; + }; + const auto IsIterationNode = [](const NodePtr &n) { + return kNextIterationOpTypes.count(n->GetType()) > 0; + }; + + if (std::any_of(ctrl_nodes.begin(), ctrl_nodes.end(), IsEnterNode)) { + // Enter --> StreamActive --> StreamMerge + return CreateMergeEnterGroup(node, node_item); + } else if (std::any_of(ctrl_nodes.begin(), ctrl_nodes.end(), IsIterationNode)) { + // NextIteration --> StreamActive {-->} StreamMerge + return CreateMergeIterationGroup(node, node_item); } + return SUCCESS; } @@ -2412,11 +2479,8 @@ Status HybridModelBuilder::CreateStreamSwitchGroup(const NodePtr &node, NodeItem // Consider as two groups, group[0] set empty for false, group[1] for true. node_item->switch_groups_.resize(kStreamSwitchNum); - const auto &out_ctrl_anchor = node->GetOutControlAnchor(); - for (const auto &peer_in_anchor : out_ctrl_anchor->GetPeerInControlAnchors()) { - const auto &dst_node = peer_in_anchor->GetOwnerNode(); + for (const auto &dst_node : node->GetOutControlNodes()) { GE_CHECK_NOTNULL(dst_node); - NodeItem *dst_node_item = nullptr; GE_CHK_STATUS_RET(GetOrCreateNodeItem(dst_node, &dst_node_item), "[%s] failed to get or create node item", dst_node->GetName().c_str()); @@ -2443,20 +2507,17 @@ Status HybridModelBuilder::CreateStreamSwitchNGroup(const NodePtr &node, NodeIte } node_item->switch_groups_.resize(batch_num); - const auto &out_ctrl_anchor = node->GetOutControlAnchor(); - for (const auto &peer_in_anchor : out_ctrl_anchor->GetPeerInControlAnchors()) { - const auto &dst_node = peer_in_anchor->GetOwnerNode(); + for (const auto &dst_node : node->GetOutControlNodes()) { GE_CHECK_NOTNULL(dst_node); - std::string batch_label; - if (!AttrUtils::GetStr(node->GetOpDesc(), ATTR_NAME_BATCH_LABEL, batch_label)) { - GELOGE(INTERNAL_ERROR, "[%s] Get ATTR_NAME_BATCH_LABEL failed", node->GetName().c_str()); + if (!AttrUtils::GetStr(dst_node->GetOpDesc(), ATTR_NAME_BATCH_LABEL, batch_label)) { + GELOGE(INTERNAL_ERROR, "[%s] Get ATTR_NAME_BATCH_LABEL failed", dst_node->GetName().c_str()); return INTERNAL_ERROR; } std::string::size_type pos = batch_label.rfind("_"); if (pos == std::string::npos) { - GELOGW("[%s] Separator not found in batch label: %s.", node->GetName().c_str(), batch_label.c_str()); + GELOGW("[%s] Separator not found in batch label: %s.", dst_node->GetName().c_str(), batch_label.c_str()); continue; } @@ -2482,7 +2543,7 @@ Status HybridModelBuilder::CreateNextIterationGroup(const NodePtr &node, NodeIte return INTERNAL_ERROR; } - return SUCCESS; + return CreateNormalNodeGroup(node, node_item); } Status HybridModelBuilder::CreateSwitchGroup(const NodePtr &node, NodeItem *node_item) { @@ -2491,11 +2552,8 @@ Status HybridModelBuilder::CreateSwitchGroup(const NodePtr &node, NodeItem *node return INTERNAL_ERROR; } - const auto &out_ctrl_anchor = node->GetOutControlAnchor(); - for (const auto &peer_in_anchor : out_ctrl_anchor->GetPeerInControlAnchors()) { - const auto &dst_node = peer_in_anchor->GetOwnerNode(); + for (const auto &dst_node : node->GetOutControlNodes()) { GE_CHECK_NOTNULL(dst_node); - NodeItem *dst_node_item = nullptr; GE_CHK_STATUS_RET(GetOrCreateNodeItem(dst_node, &dst_node_item), "[%s] failed to get or create node item", dst_node->GetName().c_str()); @@ -2505,11 +2563,8 @@ Status HybridModelBuilder::CreateSwitchGroup(const NodePtr &node, NodeItem *node // Group switch flow by out put data. node_item->switch_groups_.resize(SWITCH_OUTPUT_NUM); for (uint32_t i = 0; i < SWITCH_OUTPUT_NUM; ++i) { - const auto &out_anchor = node->GetOutDataAnchor(i); - for (const auto &peer_in_anchor : out_anchor->GetPeerInDataAnchors()) { - const auto &dst_node = peer_in_anchor->GetOwnerNode(); + for (const auto &dst_node : node->GetOutDataNodes()) { GE_CHECK_NOTNULL(dst_node); - NodeItem *dst_node_item = nullptr; GE_CHK_STATUS_RET(GetOrCreateNodeItem(dst_node, &dst_node_item), "[%s] failed to get or create node item", dst_node->GetName().c_str()); diff --git a/ge/hybrid/model/hybrid_model_builder.h b/ge/hybrid/model/hybrid_model_builder.h index ad288317..d0ee54ed 100644 --- a/ge/hybrid/model/hybrid_model_builder.h +++ b/ge/hybrid/model/hybrid_model_builder.h @@ -99,6 +99,8 @@ class HybridModelBuilder { Status BuildProfilingControl(GraphItem &graph_item, const std::map> &nodes); Status BuildControlFlowGroup(GraphItem &graph_item, const NodePtr &node, NodeItem *node_item); Status CreateNormalNodeGroup(const NodePtr &node, NodeItem *node_item); + Status CreateMergeEnterGroup(const NodePtr &node, NodeItem *node_item); + Status CreateMergeIterationGroup(const NodePtr &node, NodeItem *node_item); Status CreateStreamActiveGroup(const NodePtr &node, NodeItem *node_item); Status CreateStreamSwitchGroup(const NodePtr &node, NodeItem *node_item); Status CreateStreamSwitchNGroup(const NodePtr &node, NodeItem *node_item); diff --git a/ge/hybrid/model/node_item.cc b/ge/hybrid/model/node_item.cc index c6adce00..7054fd46 100644 --- a/ge/hybrid/model/node_item.cc +++ b/ge/hybrid/model/node_item.cc @@ -34,8 +34,8 @@ const std::set kControlOpTypes{ }; const std::set kControlFlowOpTypes{ - STREAMACTIVE, STREAMSWITCH, STREAMSWITCHN, LABELGOTO, LABELGOTOEX, LABELSWITCH, LABELSWITCHBYINDEX, - NEXTITERATION, REFNEXTITERATION + STREAMACTIVE, STREAMSWITCH, STREAMSWITCHN, NEXTITERATION, REFNEXTITERATION, EXIT, REFEXIT, + LABELGOTO, LABELGOTOEX, LABELSWITCH, LABELSWITCHBYINDEX }; const std::set kMergeOpTypes{ @@ -401,6 +401,11 @@ void NodeItem::SetDataSend(NodeItem *node_item, int anchor_index) { if (is_root_node_) { node_item->root_data_.emplace(this); } + // If Enter feed Not Merge, take as root Node. + if ((kEnterOpTypes.count(node_type) > 0) && (node_item->node_type != STREAMMERGE)) { + node_item->root_data_.emplace(this); + node_item->enter_inside_.emplace(anchor_index); + } GELOGI("Node[%s] will control node[%s]", NodeName().c_str(), node_item->NodeName().c_str()); } @@ -416,10 +421,31 @@ void NodeItem::SetCtrlSend(NodeItem *node_item, uint32_t switch_index) { if (is_root_node_) { node_item->root_ctrl_.emplace(this); } - + // If Enter feed control signal, take as root Node. + if (kEnterOpTypes.count(node_type) > 0) { + node_item->root_ctrl_.emplace(this); + } GELOGI("Node[%s] will control node[%s]", NodeName().c_str(), node_item->NodeName().c_str()); } +void NodeItem::SetMergeCtrl(NodeItem *node_item, uint32_t merge_index) { + if (merge_index >= switch_groups_.size()) { + GELOGE(FAILED, "[%s] group size: %zu, merge index: %u", NodeName().c_str(), switch_groups_.size(), merge_index); + return; + } + + // this is StreamMerge node, node_item is StreamActive node. + std::vector &switch_group = switch_groups_[merge_index]; + switch_group.emplace_back(node_item); + + node_item->ctrl_send_.emplace(this); + GELOGI("Node[%s] will control node[%s]", node_item->NodeName().c_str(), NodeName().c_str()); +} + +size_t NodeItem::GetMergeCtrl(uint32_t merge_index) const { + return ((node_type == STREAMMERGE) && (merge_index < switch_groups_.size())) ? switch_groups_[merge_index].size() : 0; +} + OptionalMutexGuard::OptionalMutexGuard(std::mutex *mutex, const string &name) : mu_(mutex), name_(name) { if (mu_ != nullptr) { GELOGD("lock for %s", name_.c_str()); diff --git a/ge/hybrid/model/node_item.h b/ge/hybrid/model/node_item.h index 606e58fe..67f92868 100644 --- a/ge/hybrid/model/node_item.h +++ b/ge/hybrid/model/node_item.h @@ -98,6 +98,8 @@ struct NodeItem { void SetDataSend(NodeItem *node_item, int anchor_index); void SetCtrlSend(NodeItem *node_item, uint32_t switch_index); + void SetMergeCtrl(NodeItem *node_item, uint32_t merge_index); + size_t GetMergeCtrl(uint32_t merge_index) const; OptionalMutexGuard MutexGuard(const std::string &name) const { return OptionalMutexGuard(copy_mu_.get(), name + "_" + node_name); @@ -140,6 +142,7 @@ struct NodeItem { std::set ctrl_send_; // Send ctrl notify to std::set ctrl_recv_; // Recv ctrl notify from std::vector> switch_groups_; // Send ctrl notify to + std::set enter_inside_; // Enter feed loop inside Node, Not cross Merge. std::shared_ptr kernel_task; std::unique_ptr fused_subgraph; diff --git a/ge/hybrid/node_executor/aicore/aicore_node_executor.cc b/ge/hybrid/node_executor/aicore/aicore_node_executor.cc index 29ae831c..7ebb9e39 100755 --- a/ge/hybrid/node_executor/aicore/aicore_node_executor.cc +++ b/ge/hybrid/node_executor/aicore/aicore_node_executor.cc @@ -41,9 +41,6 @@ AiCoreNodeTask::AiCoreNodeTask(std::vector> &&task Status AiCoreNodeExecutor::Initialize() { compiler_ = TaskCompilerFactory::GetInstance().GetTaskCompiler(); - if (compiler_ != nullptr) { - GE_CHK_STATUS_RET(compiler_->Initialize(), "[Init][TaskCompiler] failed."); - } return SUCCESS; } diff --git a/ge/hybrid/node_executor/aicore/aicore_op_task.cc b/ge/hybrid/node_executor/aicore/aicore_op_task.cc index 40118af3..8cd24bd1 100644 --- a/ge/hybrid/node_executor/aicore/aicore_op_task.cc +++ b/ge/hybrid/node_executor/aicore/aicore_op_task.cc @@ -420,9 +420,8 @@ Status AiCoreOpTask::CalcTilingInfo(const NodePtr &node, OpRunInfo &tiling_info) } Status AiCoreOpTask::UpdateArgs(TaskContext &task_context) { - size_t expected_arg_count = task_context.NumInputs() + task_context.NumOutputs() + - task_context.NumWorkspaces() - - output_indices_to_skip_.size(); + size_t expected_arg_count = task_context.NumInputs() + task_context.NumOutputs() + task_context.NumWorkspaces() - + output_indices_to_skip_.size(); if (tiling_buffer_ != nullptr) { ++expected_arg_count; } diff --git a/ge/hybrid/node_executor/aicore/aicore_task_compiler.cc b/ge/hybrid/node_executor/aicore/aicore_task_compiler.cc index 742b3ca2..0cdea5d5 100755 --- a/ge/hybrid/node_executor/aicore/aicore_task_compiler.cc +++ b/ge/hybrid/node_executor/aicore/aicore_task_compiler.cc @@ -31,6 +31,11 @@ REGISTER_TASK_COMPILER(AiCoreTaskCompiler); std::mutex AiCoreTaskCompiler::mu_; Status AiCoreTaskCompiler::Initialize() { + std::lock_guard lk(mu_); + if (is_initialized_) { + return SUCCESS; + } + auto ge_lib = GELib::GetInstance(); GE_CHECK_NOTNULL(ge_lib); if (!ge_lib->InitFlag()) { @@ -41,6 +46,7 @@ Status AiCoreTaskCompiler::Initialize() { auto &kernel_manager = ge_lib->OpsKernelManagerObj(); aic_kernel_store_ = kernel_manager.GetOpsKernelInfoStore("AIcoreEngine"); GE_CHECK_NOTNULL(aic_kernel_store_); + is_initialized_ = true; return SUCCESS; } @@ -57,6 +63,13 @@ Status AiCoreTaskCompiler::DoCompileOp(const NodePtr &node) const { } Status AiCoreTaskCompiler::CompileOp(const NodePtr &node, std::vector &tasks) { + Status ret = Initialize(); + if (ret != SUCCESS) { + GELOGE(FAILED, "[Check][State][%s] Offline inference not support online compile.", node->GetName().c_str()); + REPORT_INNER_ERROR("E19999", "[%s] Offline inference not support online compile.", node->GetName().c_str()); + return ret; + } + GE_CHECK_NOTNULL(node); GELOGI("AiCoreTaskCompiler(%s) CompileOp Start.", node->GetName().c_str()); diff --git a/ge/hybrid/node_executor/aicore/aicore_task_compiler.h b/ge/hybrid/node_executor/aicore/aicore_task_compiler.h index b6dfd82b..4cb4dc58 100755 --- a/ge/hybrid/node_executor/aicore/aicore_task_compiler.h +++ b/ge/hybrid/node_executor/aicore/aicore_task_compiler.h @@ -34,6 +34,7 @@ class AiCoreTaskCompiler : public TaskCompiler { Status DoCompileOp(const NodePtr &node) const; Status DoGenerateTask(const Node &node, std::vector &tasks); OpsKernelInfoStorePtr aic_kernel_store_; + bool is_initialized_ = false; static std::mutex mu_; }; } // namespace hybrid diff --git a/ge/hybrid/node_executor/compiledsubgraph/known_node_executor.cc b/ge/hybrid/node_executor/compiledsubgraph/known_node_executor.cc index c800e93d..8b839849 100755 --- a/ge/hybrid/node_executor/compiledsubgraph/known_node_executor.cc +++ b/ge/hybrid/node_executor/compiledsubgraph/known_node_executor.cc @@ -101,18 +101,20 @@ Status KnownNodeTask::Init(TaskContext &context) { GE_CHK_STATUS_RET(context.AllocateOutputs(), "[Allocate][Outputs] failed for %s.", context.GetNodeName()); // allocate mem base void *buffer = nullptr; - if (davinci_model_->TotalMemSize() != 0) { + int64_t total_useful_size = 0; + GE_CHK_STATUS_RET(davinci_model_->GetTotalMemSizeExcludeZeroCopy(total_useful_size), + "[Get][TotalMemSizeExcludeZeroCopy] failed."); + if (total_useful_size != 0) { RECORD_EXECUTION_EVENT(context.GetExecutionContext(), context.GetNodeName(), "[KnownNodeTask_AllocateWorkspace] Start"); - GE_CHK_STATUS_RET(context.AllocateWorkspace(davinci_model_->TotalMemSize(), &buffer, - davinci_model_->GetRuntimeParam().mem_base), + GE_CHK_STATUS_RET(context.AllocateWorkspace(total_useful_size, &buffer, davinci_model_->GetRuntimeParam().mem_base), "[Allocate][Workspace] failed for %s.", context.GetNodeName()); RECORD_EXECUTION_EVENT(context.GetExecutionContext(), context.GetNodeName(), - "[KnownNodeTask_AllocateWorkspace] End, size %zu", davinci_model_->TotalMemSize()); + "[KnownNodeTask_AllocateWorkspace] End, size %ld", total_useful_size); // update mem base davinci_model_->UpdateMemBase(static_cast(buffer)); - GELOGI("KnownNodeTask::Init mem base is %p, size %lu.", - davinci_model_->GetRuntimeParam().mem_base, davinci_model_->GetRuntimeParam().mem_size); + GELOGI("KnownNodeTask::Init mem base is %p, size %ld.", + davinci_model_->GetRuntimeParam().mem_base, total_useful_size); } GE_CHK_STATUS_RET(ModelManager::GetInstance()->DestroyAicpuKernel(davinci_model_->GetSessionId(), davinci_model_->Id(), @@ -154,8 +156,8 @@ Status KnownNodeTask::InitDavinciModel(const HybridModel &model, TensorBuffer *w weight_size = weight_buffer->GetSize(); } GELOGD("Start to init davinci model, weight size = %zu", weight_size); - GE_CHK_STATUS_RET(DoInitDavinciModel(weight, weight_size), "[Init][Model] Failed to init davinci model."); - GELOGD("[Init][Model] success"); + GE_CHK_STATUS_RET(DoInitDavinciModel(weight, weight_size), "[Init][DavinciModel] Failed to init davinci model."); + GELOGD("[Init][DavinciModel] success"); return SUCCESS; } diff --git a/ge/hybrid/node_executor/ge_local/ge_local_node_executor.cc b/ge/hybrid/node_executor/ge_local/ge_local_node_executor.cc index 43a4ca84..f6c0cf79 100755 --- a/ge/hybrid/node_executor/ge_local/ge_local_node_executor.cc +++ b/ge/hybrid/node_executor/ge_local/ge_local_node_executor.cc @@ -37,7 +37,11 @@ const std::map> {BROADCASTGRADIENTARGS, {}} }; -const std::set DependInputShapeTask::depend_input_shape_ops_ = {SHAPE, SHAPEN, RANK, SIZE, NOOP}; +const std::set DependInputShapeTask::depend_input_shape_ops_ = {SHAPE, SHAPEN, RANK, SIZE}; + +const std::set ConstantNodeTask::constant_like_task_ops_ = {CONSTANT, CONSTANTOP, VARIABLE}; + +const std::set NoOpNodeTask::control_only_task_ops_ = {NOOP, CONTROLTRIGGER}; Status RefInputTask::UpdateArgs(TaskContext &) { // no need update args @@ -244,7 +248,7 @@ Status GeLocalNodeExecutor::LoadTask(const HybridModel &model, node->GetName().c_str(), node_type.c_str()); return MEMALLOC_FAILED; } - } else if (node_type == CONSTANT || node_type == CONSTANTOP || node_type == VARIABLE) { + } else if (ConstantNodeTask::IsBelong(node_type)) { GELOGI("node %s type %s, use ConstantNodeTask.", node->GetName().c_str(), node_type.c_str()); auto tensor = model.GetTensor(node); if (tensor == nullptr) { @@ -252,9 +256,16 @@ Status GeLocalNodeExecutor::LoadTask(const HybridModel &model, GELOGE(INTERNAL_ERROR, "[Get][Tensor] failed for name: %s", node->GetName().c_str()); return INTERNAL_ERROR; } - task = MakeShared(tensor); GE_CHECK_NOTNULL(task); + } else if (NoOpNodeTask::IsBelong(node_type)) { + GELOGI("node %s type %s , use NoOpNodeTask.", node->GetName().c_str(), node_type.c_str()); + task = MakeShared(); + if (task == nullptr) { + REPORT_CALL_ERROR("E19999", "Create NoOpNodeTask failed for NoOp node %s.", node->GetName().c_str()); + GELOGE(MEMALLOC_FAILED, "[Create][NoOpNodeTask]failed for NoOp node %s.", node->GetName().c_str()); + return MEMALLOC_FAILED; + } } else { GELOGE(UNSUPPORTED, "node %s type %s is not support in GeLocalNodeExecutor now.", node->GetName().c_str(), node_type.c_str()); @@ -280,5 +291,25 @@ Status ConstantNodeTask::ExecuteAsync(TaskContext &context, std::function 0; +} + +Status NoOpNodeTask::UpdateArgs(TaskContext &context) { + // no need to update args + return SUCCESS; +} + +Status NoOpNodeTask::ExecuteAsync(TaskContext &context, std::function done_callback) { + GELOGD("[%s] Start execute.", context.GetNodeName()); + GE_CHK_STATUS_RET(context.TryExecuteCallback(done_callback)); + GELOGD("[%s] Done execute successfully.", context.GetNodeName()); + return SUCCESS; +} + +bool NoOpNodeTask::IsBelong(const std::string &op_type) { + return control_only_task_ops_.count(op_type) > 0; +} } // namespace hybrid } // namespace ge diff --git a/ge/hybrid/node_executor/ge_local/ge_local_node_executor.h b/ge/hybrid/node_executor/ge_local/ge_local_node_executor.h index c8d64d09..4b734ef3 100644 --- a/ge/hybrid/node_executor/ge_local/ge_local_node_executor.h +++ b/ge/hybrid/node_executor/ge_local/ge_local_node_executor.h @@ -75,11 +75,25 @@ class ConstantNodeTask : public NodeTask { Status UpdateArgs(TaskContext &context) override; Status ExecuteAsync(TaskContext &context, std::function done_callback) override; + static bool IsBelong(const std::string &op_type); private: + static const std::set constant_like_task_ops_; const TensorValue *tensor_; }; +class NoOpNodeTask : public NodeTask { + public: + explicit NoOpNodeTask() = default; + ~NoOpNodeTask() = default; + Status UpdateArgs(TaskContext &context) override; + Status ExecuteAsync(TaskContext &context, std::function done_callback) override; + static bool IsBelong(const std::string &op_type); + + private: + static const std::set control_only_task_ops_; +}; + class GeLocalNodeExecutor : public NodeExecutor { public: diff --git a/ge/hybrid/node_executor/hccl/hccl_node_executor.cc b/ge/hybrid/node_executor/hccl/hccl_node_executor.cc index 20684194..c46d5080 100644 --- a/ge/hybrid/node_executor/hccl/hccl_node_executor.cc +++ b/ge/hybrid/node_executor/hccl/hccl_node_executor.cc @@ -314,21 +314,26 @@ Status RdmaNodeTask::ExecuteAsync(TaskContext &context, std::function do return SUCCESS; } + rtEvent_t evt = nullptr; + if (context.GetExecutionContext()->hccl_stream != nullptr) { + GE_CHK_RT_RET(rtEventCreateWithFlag(&evt, 0x01)); + GE_CHK_RT_RET(rtStreamWaitEvent(context.GetExecutionContext()->hccl_stream, evt)); + } TaskContext *p_ctx = &context; - auto callback = [p_ctx, done_callback](HcclResult status) { + auto callback = [p_ctx, done_callback, evt](HcclResult status) { if (status != HCCL_SUCCESS) { GELOGE(HCCL_E_INTERNAL, "[Call][HcomExcutorInitialize] failed for node:%s(%s), ret: 0x%X", p_ctx->GetNodeName(), p_ctx->GetNodeItem().NodeType().c_str(), status); p_ctx->SetStatus(FAILED); } done_callback(); + if (evt != nullptr) { + GE_CHK_RT_RET(rtEventRecord(evt, nullptr)); + GE_CHK_RT_RET(rtEventDestroy(evt)); + } GELOGI("rdma callback success."); }; - std::string executor_type = context.GetNodeItem().NodeType(); - if (kRdmaScatterTypes.count(context.GetNodeItem().NodeType()) > 0) { - executor_type = context.GetNodeItem().NodeType() == HCOMREMOTEREFREAD ? HCOMREMOTEREAD : HCOMREMOTEWRITE; - } HcclResult hccl_ret = HcomExecEnqueueRemoteAccess(context.GetNodeItem().NodeType(), addr_infos, callback); if (hccl_ret != HCCL_SUCCESS) { GELOGE(HCCL_E_INTERNAL, "[Call][HcomExecEnqueueRemoteAccess] failed for node:%s(%s), ret: 0x%X", diff --git a/ge/hybrid/node_executor/rts/rts_node_task.cc b/ge/hybrid/node_executor/rts/rts_node_task.cc index f6d6ddb6..5ad8eaf4 100644 --- a/ge/hybrid/node_executor/rts/rts_node_task.cc +++ b/ge/hybrid/node_executor/rts/rts_node_task.cc @@ -20,6 +20,7 @@ #include "graph/debug/ge_attr_define.h" #include "graph/utils/tensor_utils.h" #include "graph/utils/type_utils.h" +#include "graph/utils/node_utils.h" #include "common/ge/ge_util.h" #include "common/op/ge_op_utils.h" @@ -201,6 +202,13 @@ Status PassThroughNodeTask::ExecuteAsync(TaskContext &task_context, std::functio GE_CHECK_NOTNULL(in_x); GE_CHK_STATUS_RET_NOLOG(task_context.SetOutput(0, *in_x)); // y + const auto &node_state = task_context.GetNodeState(); + if (kNextIterationOpTypes.count(node_state->GetType()) > 0) { + node_state->RunLoopNext(); + } else if (kExitOpTypes.count(node_state->GetType()) > 0) { + node_state->RunLoopExit(); + } + if (done_callback) { GE_CHK_STATUS_RET(task_context.RegisterCallback(done_callback)); } diff --git a/ge/hybrid/node_executor/rts/rts_task_factory.h b/ge/hybrid/node_executor/rts/rts_task_factory.h index a2d2bf56..c86f55d6 100644 --- a/ge/hybrid/node_executor/rts/rts_task_factory.h +++ b/ge/hybrid/node_executor/rts/rts_task_factory.h @@ -61,6 +61,6 @@ class RtsTaskFactory { REGISTER_RTS_TASK_CREATOR_UNIQ_HELPER(__COUNTER__, task_type, task_clazz) #define REGISTER_RTS_TASK_CREATOR_UNIQ_HELPER(ctr, type, clazz) \ - RtsTaskFactory::RtsTaskRegistrar g_##type##_Creator##ctr(type, []()-> RtsNodeTaskPtr { return MakeShared(); }) + RtsTaskFactory::RtsTaskRegistrar g_##type##_Creator##ctr(type, []()->RtsNodeTaskPtr { return MakeShared(); }) #endif // GE_HYBRID_NODE_EXECUTOR_RTS_TASK_FACTORY_H_ diff --git a/ge/hybrid/node_executor/task_context.cc b/ge/hybrid/node_executor/task_context.cc index 59250d8c..14eb1222 100644 --- a/ge/hybrid/node_executor/task_context.cc +++ b/ge/hybrid/node_executor/task_context.cc @@ -489,6 +489,11 @@ void TaskContext::ReleaseInputsAndOutputs() { } void TaskContext::ReleaseInput(int index) { + if (node_item_->enter_inside_.count(index) > 0) { + GELOGD("[%s] Tensor of input[%d] is enter, keep it", GetNodeName(), index); + return; + } + auto input_tensor = MutableInput(index); if (input_tensor != nullptr) { input_tensor->Destroy(); diff --git a/ge/ir_build/ge_ir_build.cc b/ge/ir_build/ge_ir_build.cc index c9dfac07..c7e9522b 100644 --- a/ge/ir_build/ge_ir_build.cc +++ b/ge/ir_build/ge_ir_build.cc @@ -133,6 +133,15 @@ static graphStatus CheckGlobalOptions(std::map &global ? "force_fp16" : global_options[ge::ir_option::PRECISION_MODE]; global_options[ge::ir_option::PRECISION_MODE] = precision_mode; + // check modify_mixlist + std::string modify_mixlist = global_options.find(ge::ir_option::MODIFY_MIXLIST) == + global_options.end() + ? "" + : global_options[ge::ir_option::MODIFY_MIXLIST]; + if (ge::CheckModifyMixlistParamValid(precision_mode, modify_mixlist) != ge::SUCCESS) { + return ge::GRAPH_PARAM_INVALID; + } + global_options[ge::ir_option::MODIFY_MIXLIST] = modify_mixlist; return GRAPH_SUCCESS; } @@ -251,17 +260,26 @@ class Impl { omg_context_.dynamic_batch_size.clear(); omg_context_.dynamic_image_size.clear(); omg_context_.dynamic_dims.clear(); + omg_context_.user_attr_index_valid = false; }; ~Impl() { (void)generator_.Finalize(); }; + graphStatus GetSupportedOptions(const std::map &in, + std::map &out); graphStatus CheckOptions(const std::map &options); graphStatus CreateInputsForIRBuild(const ge::Graph &graph, vector &inputs); graphStatus UpdateDataOpAttr(const Graph &graph); + graphStatus CheckDataOpAttrIndexValid(const Graph &graph, const std::string &input_shape_range); graphStatus Init(const Graph &graph, const std::map &options); graphStatus BuildModel(const Graph &graph, const std::map &options, ModelBufferData &ge_models); graphStatus InitDomiOmgContext(const string &input_shape, const string &input_format, const string &net_format, bool is_dynamic_input); + graphStatus GetInputShapeRange(const string &input_shape_range, + std::map>> &name_shape_range_map, + std::vector>> &index_shape_range_map); static graphStatus InferShapePrepare(const ComputeGraphPtr &compute_graph); + bool GetUsrAttrIndexValidFlag(); + bool IsAttrIndexSetByUser(const ComputeGraphPtr &compute_graph, size_t &data_num, vector &attr_index); void SetRtSocVersion(); void UpdateThreadContext(); void LoadOpsProto(); @@ -288,11 +306,113 @@ graphStatus Impl::InferShapePrepare(const ComputeGraphPtr &compute_graph) { return GRAPH_SUCCESS; } +bool Impl::GetUsrAttrIndexValidFlag() { + return omg_context_.user_attr_index_valid; +} + +bool Impl::IsAttrIndexSetByUser(const ComputeGraphPtr &compute_graph, + size_t &data_num, + vector &attr_index) { + bool all_zero_flag = true; + for (ge::NodePtr &input_node : compute_graph->GetDirectNode()) { + GE_CHECK_NOTNULL(input_node); + ge::OpDescPtr op = input_node->GetOpDesc(); + GE_CHECK_NOTNULL(op); + if (op->GetType() == DATA) { + data_num++; + GeAttrValue::INT index = 0; + if (AttrUtils::GetInt(op, ATTR_NAME_INDEX, index)) { + if (index != 0) { + all_zero_flag = false; + } + attr_index.push_back(index); + } else { + GELOGW("[Get][AttrIndex] Get index[%ld] failed for op[%s].", index, op->GetName().c_str()); + } + } + } + if (data_num > 1 && attr_index.size() == data_num && all_zero_flag) { + GELOGI("Attr indexes are not set by user."); + return false; + } + return true; +} + +graphStatus Impl::GetInputShapeRange(const string &input_shape_range, + std::map>> &name_shape_range_map, + std::vector>> &index_shape_range_map) { + if (input_shape_range.empty()) { + GELOGI("Input shape range is empty."); + return GRAPH_SUCCESS; + } + Status ret = GRAPH_PARAM_INVALID; + if (input_shape_range.find(":") != string::npos) { + ret = ParseInputShapeRange(input_shape_range, name_shape_range_map); + } else { + ret = ParseInputShapeRange(input_shape_range, index_shape_range_map); + } + if (ret != SUCCESS) { + GELOGE(GRAPH_PARAM_INVALID, "[Parse][InputShapeRange] parse shape range[%s] failed.", input_shape_range.c_str()); + return GRAPH_PARAM_INVALID; + } + return GRAPH_SUCCESS; +} + +graphStatus Impl::CheckDataOpAttrIndexValid(const Graph &graph, const std::string &input_shape_range) { + auto compute_graph = ge::GraphUtils::GetComputeGraph(graph); + GE_CHECK_NOTNULL(compute_graph); + + // when set input shape range by index, user must set data attr index, eg. "[1, 3, 3, -1],[1, 3~5, 6, -1]" + bool index_input_shape_range_flag = !input_shape_range.empty() && (input_shape_range.find(":") == string::npos); + size_t data_num = 0; + vector attr_index; + if (!IsAttrIndexSetByUser(compute_graph, data_num, attr_index)) { + if (index_input_shape_range_flag) { + std::string situation = "Data op index"; + std::string reason = "it must be set by user, total data op num[" + std::to_string(data_num) + "], " + "when set input shape range by index."; + REPORT_INPUT_ERROR("E19025", std::vector({"situation", "reason"}), + std::vector({situation, reason})); + GELOGE(GRAPH_FAILED, "[Check][AttrIndex] Data op index is not set, total data op num[%ld], " + "when set input shape range by index.", data_num); + return GRAPH_FAILED; + } + return GRAPH_SUCCESS; + } + + omg_context_.user_attr_index_valid = true; + for (size_t i = 0; i < data_num; ++i) { + if (std::find(attr_index.begin(), attr_index.end(), i) == attr_index.end()) { + omg_context_.user_attr_index_valid = false; + if (index_input_shape_range_flag) { + std::string situation = "Data op index[" + std::to_string(i) + "]"; + std::string reason = "it must be set by user, total data op num[" + std::to_string(data_num) + "], " + "when set input shape range by index"; + REPORT_INPUT_ERROR("E19025", std::vector({"situation", "reason"}), + std::vector({situation, reason})); + GELOGE(GRAPH_FAILED, "[Check][AttrIndex] Attr index [%ld] is not set, total data op num[%ld], " + "when set input shape range by index", i, data_num); + return GRAPH_FAILED; + } else { + GELOGW("[Check][AttrIndex] Attr index [%ld] is not set, total data op num[%ld].", i, data_num); + } + } + } + GELOGI("Data op attr indexes are set by user and valid."); + return GRAPH_SUCCESS; +} + graphStatus Impl::UpdateDataOpAttr(const Graph &graph) { GELOGD("Enter Update Data Attr Process!"); std::string input_shape = (options_.find(kInputShape) == options_.end()) ? "" : options_[kInputShape]; std::string input_shape_range = (options_.find(kInputShapeRange) == options_.end()) ? "" : options_[kInputShapeRange]; + graphStatus ret = CheckDataOpAttrIndexValid(graph, input_shape_range); + if (ret != GRAPH_SUCCESS) { + GELOGE(GRAPH_FAILED, "[Check][DataOpAttrIndex] fail, shape range[%s].", input_shape_range.c_str()); + return GRAPH_FAILED; + } + map> shape_map; vector>> user_shape_map; if (!input_shape.empty()) { @@ -301,20 +421,13 @@ graphStatus Impl::UpdateDataOpAttr(const Graph &graph) { } std::map>> name_shape_range_map; std::vector>> index_shape_range_map; - if (!input_shape_range.empty()) { - Status ret = GRAPH_PARAM_INVALID; - if (input_shape_range.find(":") != string::npos) { - ret = ParseInputShapeRange(input_shape_range, name_shape_range_map); - } else { - ret = ParseInputShapeRange(input_shape_range, index_shape_range_map); - } - if (ret != SUCCESS) { - GELOGE(GRAPH_PARAM_INVALID, "[Parse][InputShapeRange] parse shape range[%s] failed.", input_shape_range.c_str()); - return GRAPH_PARAM_INVALID; - } - } auto compute_graph = ge::GraphUtils::GetComputeGraph(graph); GE_CHECK_NOTNULL(compute_graph); + ret = GetInputShapeRange(input_shape_range, name_shape_range_map, index_shape_range_map); + if (ret != GRAPH_SUCCESS) { + GELOGE(GRAPH_FAILED, "[Get][InputShapeRange] fail, shape range[%s].", input_shape_range.c_str()); + return GRAPH_FAILED; + } for (ge::NodePtr &input_node : compute_graph->GetDirectNode()) { GE_CHECK_NOTNULL(input_node); ge::OpDescPtr op = input_node->GetOpDesc(); @@ -338,19 +451,29 @@ graphStatus Impl::UpdateDataOpAttr(const Graph &graph) { return GRAPH_SUCCESS; } -graphStatus Impl::CheckOptions(const std::map &options) { - for (auto &ele : options) { +graphStatus Impl::GetSupportedOptions(const std::map &in, + std::map &out) { + for (auto &ele : in) { auto it = ge::ir_option::ir_builder_suppported_options.find(ele.first); if (it == ge::ir_option::ir_builder_suppported_options.end()) { auto it_lx_fusion = ir_builder_supported_options_for_lx_fusion.find(ele.first); if (it_lx_fusion == ir_builder_supported_options_for_lx_fusion.end()) { GELOGE(GRAPH_PARAM_INVALID, "[Check][Options] unsupported option(%s), Please check!", - ele.first.c_str()); + ele.first.c_str()); return GRAPH_PARAM_INVALID; } } - options_.insert(ele); + out.insert(ele); + } + return GRAPH_SUCCESS; +} + +graphStatus Impl::CheckOptions(const std::map &options) { + auto ret = GetSupportedOptions(options, options_); + if (ret != GRAPH_SUCCESS) { + return ret; } + // Check options build_mode and build_step. std::string build_mode; auto it = options_.find(BUILD_MODE); @@ -378,6 +501,10 @@ graphStatus Impl::CheckOptions(const std::map &options if (it != options_.end() && (CheckDisableReuseMemoryParamValid(it->second) != GRAPH_SUCCESS)) { return GRAPH_PARAM_INVALID; } + // Check option modify_mixlist + if (ge::CheckModifyMixlistParamValid(options_) != GRAPH_SUCCESS) { + return GRAPH_PARAM_INVALID; + } // Check Input Format if (options_.find(kInputFormat) != options_.end()) { return CheckInputFormat(options_[kInputFormat]); @@ -495,7 +622,9 @@ graphStatus Impl::CreateInputsForIRBuild(const ge::Graph &graph, vectorGetOpDesc(); GE_CHECK_NOTNULL(op); if (op->GetType() == DATA) { - (void)AttrUtils::SetInt(op, ATTR_NAME_INDEX, index++); + if (!GetUsrAttrIndexValidFlag()) { + (void)AttrUtils::SetInt(op, ATTR_NAME_INDEX, index++); + } GELOGD("Data op inputDesc size: %zu", op->GetAllInputsDesc().size()); ge::GeTensorDesc tensor = op->GetInputDesc(0); string data_op_name = op->GetName(); diff --git a/ge/ir_build/option_utils.cc b/ge/ir_build/option_utils.cc index c23da519..d7fccee3 100755 --- a/ge/ir_build/option_utils.cc +++ b/ge/ir_build/option_utils.cc @@ -37,6 +37,9 @@ const size_t kMaxNDDimNum = 4; const size_t kMinNDDimNum = 1; const size_t kSquareBracketsSize = 2; const size_t kRangePairSize = 2; +const size_t kShapeRangeSize = 2; +const size_t kShapeRangeStrIndex = 2; +const size_t kShapeRangeStrSize = 3; // datatype/formats from user to GE, Unified to util interface file later const std::map kOutputTypeSupportDatatype = { {"FP32", ge::DT_FLOAT}, {"FP16", ge::DT_FLOAT16}, {"UINT8", ge::DT_UINT8}}; @@ -253,8 +256,7 @@ bool CheckDynamicImagesizeInputShapeValid(map> shape_map for (auto str : split_set) { split_dim = StringUtils::Split(str, ','); if (split_dim.size() != static_cast(kDynamicImageSizeNum)) { - ErrorManager::GetInstance().ATCReportErrMessage("E10020", {"DynamicImageSizeNum"}, - {std::to_string(kDynamicImageSizeNum)}); + ErrorManager::GetInstance().ATCReportErrMessage("E10020"); GELOGE(ge::PARAM_INVALID, "[Check][DynamicImagesizeInputShape] invalid value:%s number of dimensions of each group must be %ld.", dynamic_image_size.c_str(), kDynamicImageSizeNum); @@ -435,7 +437,7 @@ Status ParseInputShapeRange(const std::string &shape_range, std::vector>> &range) { GELOGD("Input shape range %s", shape_range.c_str()); - if (shape_range.size() < 2) { + if (shape_range.size() < kShapeRangeSize) { REPORT_INPUT_ERROR("E10048", std::vector({"shape_range", "reason", "sample"}), std::vector({shape_range, kInputShapeRangeSizeInvalid, kInputShapeRangeSample4})); GELOGE(PARAM_INVALID, "[Parse][ShapeRange] str:%s invalid, reason: %s, correct sample is %s.", @@ -452,7 +454,7 @@ Status ParseInputShapeRange(const std::string &shape_range, return PARAM_INVALID; } for (auto &shape_range_str : shape_range_set) { - if (shape_range_str.size() < 3) { + if (shape_range_str.size() < kShapeRangeStrSize) { // shape_range_str should be "[2~3,1" // or ",[2~3,1". because we should trim '[' or ',[' // so shape_range_str.size() < 3 is invalid @@ -463,7 +465,7 @@ Status ParseInputShapeRange(const std::string &shape_range, shape_range_str = shape_range_str.substr(1, shape_range_str.size()); } if (ge::StringUtils::StartWith(shape_range_str, ",")) { - shape_range_str = shape_range_str.substr(2, shape_range_str.size()); + shape_range_str = shape_range_str.substr(kShapeRangeStrIndex, shape_range_str.size()); } // parse shape_range of single input. eg. "1~20,3,3~6,-1" @@ -785,6 +787,31 @@ Status CheckImplmodeParamValid(const std::string &optypelist_for_implmode, std:: return ge::SUCCESS; } +Status CheckModifyMixlistParamValid(const std::map &options) { + std::string precision_mode; + auto it = options.find(ge::PRECISION_MODE); + if (it != options.end()) { + precision_mode = it->second; + } + it = options.find(ge::MODIFY_MIXLIST); + if (it != options.end() && CheckModifyMixlistParamValid(precision_mode, it->second) != ge::SUCCESS) { + return ge::PARAM_INVALID; + } + return ge::SUCCESS; +} + +Status CheckModifyMixlistParamValid(const std::string &precision_mode, const std::string &modify_mixlist) { + if (!modify_mixlist.empty() && precision_mode != "allow_mix_precision") { + REPORT_INPUT_ERROR("E10001", std::vector({"parameter", "value", "reason"}), + std::vector({ge::MODIFY_MIXLIST, modify_mixlist, kModifyMixlistError})); + GELOGE(ge::PARAM_INVALID, "[Check][ModifyMixlist] Failed, %s", kModifyMixlistError); + return ge::PARAM_INVALID; + } + GELOGI("Option set successfully, option_key=%s, option_value=%s", ge::MODIFY_MIXLIST.c_str(), modify_mixlist.c_str()); + + return ge::SUCCESS; +} + void PrintOptionMap(std::map &options, std::string tips) { for (auto iter = options.begin(); iter != options.end(); iter++) { std::string key = iter->first; diff --git a/ge/ir_build/option_utils.h b/ge/ir_build/option_utils.h index 44504e35..0b25bdf0 100644 --- a/ge/ir_build/option_utils.h +++ b/ge/ir_build/option_utils.h @@ -29,6 +29,8 @@ #include "graph/preprocess/multi_batch_options.h" namespace ge { +const char *const kModifyMixlistError = "modify_mixlist is assigned, please ensure that " + "precision_mode is assigned to 'allow_mix_precision'"; static std::set caffe_support_input_format = {"NCHW", "ND"}; static std::set tf_support_input_format = {"NCHW", "NHWC", "ND", "NCDHW", "NDHWC"}; static std::set onnx_support_input_format = {"NCHW", "ND", "NCDHW"}; @@ -77,6 +79,8 @@ Status CheckInsertOpConfParamValid(const std::string insert_op_conf); Status CheckDisableReuseMemoryParamValid(const std::string disable_reuse_memory); Status CheckEnableSingleStreamParamValid(const std::string enable_single_stream); Status CheckImplmodeParamValid(const std::string &optypelist_for_implmode, std::string &op_select_implmode); +Status CheckModifyMixlistParamValid(const std::map &options); +Status CheckModifyMixlistParamValid(const std::string &precision_mode, const std::string &modify_mixlist); Status CheckInputFormat(const string &input_format); Status CheckKeepTypeParamValid(const std::string &keep_dtype); void PrintOptionMap(std::map &options, std::string tips); diff --git a/ge/offline/CMakeLists.txt b/ge/offline/CMakeLists.txt index 2a0f0ff0..a520652f 100644 --- a/ge/offline/CMakeLists.txt +++ b/ge/offline/CMakeLists.txt @@ -1,12 +1,3 @@ -set(PROTO_LIST - "${METADEF_DIR}/proto/om.proto" - "${METADEF_DIR}/proto/ge_ir.proto" - "${METADEF_DIR}/proto/insert_op.proto" - "${METADEF_DIR}/proto/task.proto" -) - -protobuf_generate(ge PROTO_SRCS PROTO_HDRS ${PROTO_LIST}) - set(SRC_LIST "main.cc" "single_op_parser.cc" @@ -15,7 +6,11 @@ set(SRC_LIST ) ############ atc_atc.bin ############ -add_executable(atc_atc.bin ${SRC_LIST} ${PROTO_HDRS}) +add_executable(atc_atc.bin ${SRC_LIST}) + +add_dependencies(atc_atc.bin + graphengine_protos +) target_compile_options(atc_atc.bin PRIVATE -Werror @@ -50,7 +45,7 @@ target_include_directories(atc_atc.bin PRIVATE ${METADEF_DIR}/inc/external/register ${PARSER_DIR} ${CMAKE_BINARY_DIR} - ${CMAKE_BINARY_DIR}/proto/ge + ${CMAKE_BINARY_DIR}/proto/graphengine_protos #### yellow zone #### ${GE_CODE_DIR}/../inc ${GE_CODE_DIR}/../inc/common @@ -88,7 +83,11 @@ set_target_properties(atc_atc.bin PROPERTIES ) ############ fwk_atc.bin ############ -add_executable(fwk_atc.bin ${SRC_LIST} ${PROTO_HDRS}) +add_executable(fwk_atc.bin ${SRC_LIST}) + +add_dependencies(fwk_atc.bin + graphengine_protos +) target_compile_options(fwk_atc.bin PRIVATE -Werror @@ -123,7 +122,7 @@ target_include_directories(fwk_atc.bin PRIVATE ${METADEF_DIR}/inc/external/register ${PARSER_DIR} ${CMAKE_BINARY_DIR} - ${CMAKE_BINARY_DIR}/proto/ge + ${CMAKE_BINARY_DIR}/proto/graphengine_protos #### yellow zone #### ${GE_CODE_DIR}/../inc ${GE_CODE_DIR}/../inc/common diff --git a/ge/offline/main.cc b/ge/offline/main.cc index 8eb83010..12e39680 100755 --- a/ge/offline/main.cc +++ b/ge/offline/main.cc @@ -116,7 +116,7 @@ DEFINE_string(out_nodes, "", DEFINE_string(precision_mode, "force_fp16", "Optional; precision mode." - "Support force_fp16, allow_mix_precision, allow_fp32_to_fp16, must_keep_origin_dtype."); + "Support force_fp16, force_fp32, allow_mix_precision, allow_fp32_to_fp16, must_keep_origin_dtype."); DEFINE_string(keep_dtype, "", "Optional; config file to specify the precision used by the operator during compilation."); @@ -218,6 +218,8 @@ DEFINE_string(display_model_info, "0", "Optional; display model info"); DEFINE_string(device_id, "0", "Optional; user device id"); +DEFINE_string(modify_mixlist, "", "Optional; operator mixed precision configuration file path"); + class GFlagUtils { public: /** @@ -304,7 +306,7 @@ class GFlagUtils { "\"l1_optimize\", \"off_optimize\"\n" " --mdl_bank_path Set the path of the custom repository generated after model tuning.\n" "\n[Operator Tuning]\n" - " --precision_mode precision mode, support force_fp16(default), allow_mix_precision, " + " --precision_mode precision mode, support force_fp16(default), force_fp32, allow_mix_precision, " "allow_fp32_to_fp16, must_keep_origin_dtype.\n" " --keep_dtype Retains the precision of certain operators in inference " "scenarios by using a configuration file.\n" @@ -321,6 +323,7 @@ class GFlagUtils { " 2: Enable TBE pipe_all, generate the operator CCE file and Python-CCE mapping file " "(.json), and enable the CCE compiler -O0-g.\n" " 3: Disable debug, and keep generating kernel file (.o and .json)\n" + " --modify_mixlist Set the path of operator mixed precision configuration file.\n" "\n[Debug]\n" " --save_original_model Control whether to output original model. E.g.: true: output original model\n" " --log Generate log with level. Support debug, info, warning, error, null\n" @@ -369,6 +372,14 @@ class GFlagUtils { ge::CheckImplmodeParamValid(FLAGS_optypelist_for_implmode, FLAGS_op_select_implmode) != ge::SUCCESS, ret = ge::FAILED, "[Check][ImplMode]check optypelist_for_implmode and op_select_implmode failed!"); + + if (ge::CheckModifyMixlistParamValid(FLAGS_precision_mode, FLAGS_modify_mixlist) != ge::SUCCESS) { + ErrorManager::GetInstance().ATCReportErrMessage("E10001", {"parameter", "value", "reason"}, + {"modify_mixlist", FLAGS_modify_mixlist.c_str(), + ge::kModifyMixlistError}); + ret = ge::FAILED; + } + // No output file information passed in GE_CHK_BOOL_TRUE_EXEC_WITH_LOG( FLAGS_mode == GEN_OM_MODEL && FLAGS_output == "", @@ -1081,6 +1092,7 @@ static void SetEnvForSingleOp(std::map &options) { options.emplace(ge::MDL_BANK_PATH_FLAG, FLAGS_mdl_bank_path); options.emplace(ge::OP_BANK_PATH_FLAG, FLAGS_op_bank_path); options.emplace(ge::TUNE_DEVICE_IDS, FLAGS_device_id); + options.emplace(ge::MODIFY_MIXLIST, FLAGS_modify_mixlist); } domi::Status GenerateSingleOp(const std::string& json_file_path) { @@ -1093,9 +1105,18 @@ domi::Status GenerateSingleOp(const std::string& json_file_path) { ge::CheckImplmodeParamValid(FLAGS_optypelist_for_implmode, FLAGS_op_select_implmode) != ge::SUCCESS, return ge::FAILED, "[Check][ImplmodeParam] fail for input optypelist_for_implmode and op_select_implmode."); + if (ge::CheckModifyMixlistParamValid(FLAGS_precision_mode, FLAGS_modify_mixlist) != ge::SUCCESS) { + ErrorManager::GetInstance().ATCReportErrMessage("E10001", {"parameter", "value", "reason"}, + {"modify_mixlist", FLAGS_modify_mixlist.c_str(), + ge::kModifyMixlistError}); + return ge::FAILED; + } + std::map options; // need to be changed when ge.ini plan is done SetEnvForSingleOp(options); + // print single op option map + ge::PrintOptionMap(options, "single op option"); auto ret = ge::GELib::Initialize(options); if (ret != ge::SUCCESS) { @@ -1234,6 +1255,8 @@ domi::Status GenerateOmModel() { options.insert(std::pair(string(ge::DISPLAY_MODEL_INFO), FLAGS_display_model_info)); + options.insert(std::pair(string(ge::MODIFY_MIXLIST), FLAGS_modify_mixlist)); + // set enable scope fusion passes SetEnableScopeFusionPasses(FLAGS_enable_scope_fusion_passes); // print atc option map diff --git a/ge/plugin/engine/CMakeLists.txt b/ge/plugin/engine/CMakeLists.txt index 3aace4ac..b4ea9c52 100644 --- a/ge/plugin/engine/CMakeLists.txt +++ b/ge/plugin/engine/CMakeLists.txt @@ -6,6 +6,10 @@ set(SRC_LIST ############ libengine.so ############ add_library(engine SHARED ${SRC_LIST}) +add_dependencies(engine + graphengine_protos +) + target_compile_options(engine PRIVATE -Werror -fno-common @@ -27,7 +31,7 @@ target_include_directories(engine PRIVATE ${METADEF_DIR}/inc ${METADEF_DIR}/inc/external ${CMAKE_BINARY_DIR} - ${CMAKE_BINARY_DIR}/proto/ge + ${CMAKE_BINARY_DIR}/proto/graphengine_protos #### yellow zone #### ${GE_CODE_DIR}/../inc #### blue zone #### diff --git a/ge/session/inner_session.cc b/ge/session/inner_session.cc index 39c87107..8248eecf 100755 --- a/ge/session/inner_session.cc +++ b/ge/session/inner_session.cc @@ -35,6 +35,7 @@ #include "graph/manager/graph_mem_manager.h" #include "graph/utils/tensor_adapter.h" #include "runtime/mem.h" +#include "ir_build/option_utils.h" namespace ge { namespace { @@ -81,6 +82,11 @@ Status InnerSession::Initialize() { return ret; } + // Check option modify_mixlist + if (ge::CheckModifyMixlistParamValid(all_options) != ge::SUCCESS) { + return FAILED; + } + UpdateThreadContext(std::map{}); // session device id set here diff --git a/ge/single_op/single_op.cc b/ge/single_op/single_op.cc index 36ca1850..d09e8398 100755 --- a/ge/single_op/single_op.cc +++ b/ge/single_op/single_op.cc @@ -113,6 +113,30 @@ Status UpdateInputsBufferAddr(StreamResource *stream_resource, rtStream_t stream return SUCCESS; } +Status ModifyTensorDesc(GeTensorDesc &tensor) { + int64_t storage_format_val = static_cast(FORMAT_RESERVED); + (void)AttrUtils::GetInt(tensor, ge::ATTR_NAME_STORAGE_FORMAT, storage_format_val); + auto storage_format = static_cast(storage_format_val); + auto format = tensor.GetFormat(); + if (storage_format != FORMAT_RESERVED && storage_format != format) { + std::vector storage_shape; + if (!AttrUtils::GetListInt(tensor, ge::ATTR_NAME_STORAGE_SHAPE, storage_shape)) { + GELOGE(ACL_ERROR_GE_INTERNAL_ERROR, "[Get][storage_shape]failed while storage_format was set."); + REPORT_INNER_ERROR("E19999", "Get storage_shape failed while storage_format was set."); + return ACL_ERROR_GE_INTERNAL_ERROR; + } + + GELOGD("Storage format set. update shape to [%s], and original shape to [%s]", + GeShape(storage_shape).ToString().c_str(), tensor.GetShape().ToString().c_str()); + tensor.SetOriginShape(tensor.GetShape()); + tensor.SetOriginFormat(format); + tensor.SetShape(GeShape(storage_shape)); + tensor.SetFormat(storage_format); + } + + return SUCCESS; +} + Status InitHybridModelArgs(const std::vector &input_buffers, const std::vector &output_buffers, const std::vector &inputs_desc, @@ -126,6 +150,7 @@ Status InitHybridModelArgs(const std::vector &input_buffers, for (auto &tensor_desc : inputs_desc) { auto desc = MakeShared(tensor_desc); GE_CHECK_NOTNULL(desc); + GE_CHK_STATUS_RET_NOLOG(ModifyTensorDesc(*desc)); args.input_desc.emplace_back(desc); } return SUCCESS; diff --git a/ge/single_op/single_op_model.cc b/ge/single_op/single_op_model.cc index 6959c6b3..5b5f24a2 100755 --- a/ge/single_op/single_op_model.cc +++ b/ge/single_op/single_op_model.cc @@ -613,7 +613,7 @@ Status SingleOpModel::BuildDynamicOp(StreamResource &resource, DynamicSingleOp & single_op.num_inputs_ = data_ops_.size(); single_op.num_outputs_ = netoutput_op_->GetAllInputsSize(); GE_CHK_STATUS_RET_NOLOG(InitModelMem(resource)); - model_params_.memory_size = UINT_MAX; + model_params_.memory_size = UINT64_MAX; model_params_.graph_is_dynamic = true; auto ge_model = model_helper_.GetGeModel(); diff --git a/inc/external/ge/ge_api_types.h b/inc/external/ge/ge_api_types.h index 388f0fe0..fbd6c020 100644 --- a/inc/external/ge/ge_api_types.h +++ b/inc/external/ge/ge_api_types.h @@ -112,6 +112,7 @@ const char *const ORIGINAL_MODEL_FILE = "ge.originalModelFile"; const char *const INPUT_FP16_NODES = "ge.INPUT_NODES_SET_FP16"; const char *const OP_DEBUG_LEVEL = "ge.opDebugLevel"; const char *const PERFORMANCE_MODE = "ge.performance_mode"; +const char *const MODIFY_MIXLIST = "ge.exec.modify_mixlist"; } // namespace configure_option // Configure stream num by Session constructor options param, // its value should be int32_t type, default value is "1" @@ -323,6 +324,8 @@ const char *const INPUT_SHAPE_RANGE = "input_shape_range"; // high: need to recompile, high execute performance mode const std::string PERFORMANCE_MODE = "ge.performance_mode"; +const std::string MODIFY_MIXLIST = "ge.exec.modify_mixlist"; + // Graph run mode enum GraphRunMode { PREDICTION = 0, TRAIN }; @@ -401,6 +404,7 @@ static const char *const OP_BANK_PATH = ge::OP_BANK_PATH_FLAG.c_str(); static const char *const OP_BANK_UPDATE = ge::OP_BANK_UPDATE_FLAG.c_str(); static const char *const OP_DEBUG_LEVEL = ge::OP_DEBUG_LEVEL.c_str(); static const char *const PERFORMANCE_MODE = ge::PERFORMANCE_MODE.c_str(); +static const char *const MODIFY_MIXLIST = ge::MODIFY_MIXLIST.c_str(); // for interface: aclgrphBuildModel #ifdef __GNUC__ @@ -427,7 +431,8 @@ const std::set ir_builder_suppported_options = {INPUT_FORMAT, MDL_BANK_PATH, OP_BANK_PATH, OP_BANK_UPDATE, - PERFORMANCE_MODE}; + PERFORMANCE_MODE, + MODIFY_MIXLIST}; // for interface: aclgrphParse const std::set ir_parser_suppported_options = { @@ -453,7 +458,8 @@ const std::set global_options = {CORE_TYPE, OP_DEBUG_LEVEL, DEBUG_DIR, OP_COMPILER_CACHE_DIR, - OP_COMPILER_CACHE_MODE}; + OP_COMPILER_CACHE_MODE, + MODIFY_MIXLIST}; #endif } // namespace ir_option } // namespace ge diff --git a/inc/framework/generator/ge_generator.h b/inc/framework/generator/ge_generator.h index 24f969dd..ee51d29d 100644 --- a/inc/framework/generator/ge_generator.h +++ b/inc/framework/generator/ge_generator.h @@ -106,6 +106,7 @@ class GE_FUNC_VISIBILITY GeGenerator { bool CheckNoAicore(const ComputeGraphPtr &graph); void RemoveConst(const vector &inputs, vector &outputs); Status CheckForSingleOp(OpDescPtr &op_desc, const vector &inputs, const vector &outputs); + Status InferFormatForSingleOp(OpDescPtr &op_desc); using GeRootModelPtr = std::shared_ptr; Status SetModelNameForDump(const GeRootModelPtr &ge_root_model); diff --git a/inc/framework/omg/omg_inner_types.h b/inc/framework/omg/omg_inner_types.h index 1024f7e6..0b799bf2 100644 --- a/inc/framework/omg/omg_inner_types.h +++ b/inc/framework/omg/omg_inner_types.h @@ -125,6 +125,7 @@ struct OmgContext { std::vector getnext_nosink_nodes; bool fuzz_compile_flag = false; std::string atc_cmdline; + bool user_attr_index_valid = false; }; } // namespace ge diff --git a/inc/framework/omg/parser/model_parser.h b/inc/framework/omg/parser/model_parser.h index 8fae5556..67f9338d 100644 --- a/inc/framework/omg/parser/model_parser.h +++ b/inc/framework/omg/parser/model_parser.h @@ -37,6 +37,9 @@ using Status = domi::Status; namespace domi { using GetGraphCallback = std::function( const google::protobuf::Message *root_proto, const std::string &graph)>; + +using GetGraphCallbackV2 = std::function; + class GE_FUNC_VISIBILITY ModelParser { public: ModelParser() {} @@ -117,6 +120,30 @@ class GE_FUNC_VISIBILITY ModelParser { virtual ge::DataType ConvertToGeDataType(const uint32_t type) = 0; virtual Status ParseAllGraph(const google::protobuf::Message *root_proto, ge::ComputeGraphPtr &root_graph) = 0; + + /** + * @ingroup domi_omg + * @brief Analyze network model data + * @param [in] proto serialized network model + * @param [in|out] graph Save the network information after analysis + * @return SUCCESS + * @return Others failed + */ + virtual Status ParseProto(const std::string &serialized_proto, ge::ComputeGraphPtr &graph) { return UNSUPPORTED; } + + /** + * @ingroup domi_omg + * @brief Analyze callback model data in subgraph + * @param [in] proto serialized network model + * @param [in] callback callback of subgraph + * @param [in|out] graph Save the network information after analysis + * @return SUCCESS + * @return Others failed + */ + virtual Status ParseProtoWithSubgraph(const std::string &serialized_proto, GetGraphCallbackV2 callback, + ge::ComputeGraphPtr &graph) { + return UNSUPPORTED; + } }; } // namespace domi diff --git a/metadef b/metadef index 7cbdf957..854a3313 160000 --- a/metadef +++ b/metadef @@ -1 +1 @@ -Subproject commit 7cbdf95765133b5a5b979c2231013f7c76c3d529 +Subproject commit 854a3313eb5ff68af7300b393d48fdbfb5bf466e diff --git a/parser b/parser index 8d44bebf..39d868cb 160000 --- a/parser +++ b/parser @@ -1 +1 @@ -Subproject commit 8d44bebfeeb71b793bc7325acc95345090789e19 +Subproject commit 39d868cb961181f421eb80775a978ca519ef65cd diff --git a/scripts/build/ge_build.sh b/scripts/build/ge_build.sh new file mode 100755 index 00000000..c230ec09 --- /dev/null +++ b/scripts/build/ge_build.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +set -e +export PROJECT_HOME=${PROJECT_HOME:-$(dirname "$0")/../} +function main(){ + ${PROJECT_HOME}/build.sh "$@" +} + +main "$@" + +set +e diff --git a/scripts/check_clang_format.sh b/scripts/check_clang_format.sh old mode 100644 new mode 100755 index 2425345b..29434fd4 --- a/scripts/check_clang_format.sh +++ b/scripts/check_clang_format.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2019-2020 Huawei Technologies Co., Ltd +# Copyright 2021 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ - set -e CLANG_FORMAT=$(which clang-format) || (echo "Please install 'clang-format' tool first"; exit 1) @@ -25,10 +24,10 @@ if [[ "${version}" -lt "8" ]]; then fi CURRENT_PATH=$(pwd) -SCRIPTS_PATH=$(dirname "$0") +PROJECT_HOME=${PROJECT_HOME:-$(dirname "$0")/../} echo "CURRENT_PATH=$CURRENT_PATH" -echo "SCRIPTS_PATH=$SCRIPTS_PATH" +echo "PROJECT_HOME=$PROJECT_HOME" # print usage message function usage() @@ -81,45 +80,46 @@ function checkopts() checkopts "$@" # switch to project root path, which contains clang-format config file '.clang-format' -cd "${SCRIPTS_PATH}/.." || exit 1 +pushd "${CURRENT_PATH}" + CHECK_LIST_FILE='__checked_files_list__' + cd "${PROJECT_HOME}" || exit 1 -CHECK_LIST_FILE='__checked_files_list__' + if [ "X${mode}" == "Xall" ]; then + find src -type f -name "*" | grep "\.h$\|\.cc$" > "${CHECK_LIST_FILE}" || true + find inc -type f -name "*" | grep "\.h$\|\.cc$" >> "${CHECK_LIST_FILE}" || true + elif [ "X${mode}" == "Xchanged" ]; then + # --diff-filter=ACMRTUXB will ignore deleted files in commit + git diff --diff-filter=ACMRTUXB --name-only | grep "^inc\|^src" | grep "\.h$\|\.cc$" > "${CHECK_LIST_FILE}" || true + else # "X${mode}" == "Xlastcommit" + git diff --diff-filter=ACMRTUXB --name-only HEAD~ HEAD | grep "^inc\|^src" | grep "\.h$\|\.cc$" > "${CHECK_LIST_FILE}" || true + fi -if [ "X${mode}" == "Xall" ]; then - find src -type f -name "*" | grep "\.h$\|\.cc$" > "${CHECK_LIST_FILE}" || true - find inc -type f -name "*" | grep "\.h$\|\.cc$" >> "${CHECK_LIST_FILE}" || true -elif [ "X${mode}" == "Xchanged" ]; then - # --diff-filter=ACMRTUXB will ignore deleted files in commit - git diff --diff-filter=ACMRTUXB --name-only | grep "^inc\|^src" | grep "\.h$\|\.cc$" > "${CHECK_LIST_FILE}" || true -else # "X${mode}" == "Xlastcommit" - git diff --diff-filter=ACMRTUXB --name-only HEAD~ HEAD | grep "^inc\|^src" | grep "\.h$\|\.cc$" > "${CHECK_LIST_FILE}" || true -fi + CHECK_RESULT_FILE=__code_format_check_result__ + echo "0" > "$CHECK_RESULT_FILE" -CHECK_RESULT_FILE=__code_format_check_result__ -echo "0" > "$CHECK_RESULT_FILE" + # check format of files modified in the lastest commit + while read line; do + BASE_NAME=$(basename "${line}") + TEMP_FILE="__TEMP__${BASE_NAME}" + cp "${line}" "${TEMP_FILE}" + ${CLANG_FORMAT} -i "${TEMP_FILE}" + set +e + diff "${TEMP_FILE}" "${line}" + ret=$? + set -e + rm "${TEMP_FILE}" + if [[ "${ret}" -ne 0 ]]; then + echo "File ${line} is not formated, please format it." + echo "1" > "${CHECK_RESULT_FILE}" + break + fi + done < "${CHECK_LIST_FILE}" -# check format of files modified in the lastest commit -while read line; do - BASE_NAME=$(basename "${line}") - TEMP_FILE="__TEMP__${BASE_NAME}" - cp "${line}" "${TEMP_FILE}" - ${CLANG_FORMAT} -i "${TEMP_FILE}" - set +e - diff "${TEMP_FILE}" "${line}" - ret=$? - set -e - rm "${TEMP_FILE}" - if [[ "${ret}" -ne 0 ]]; then - echo "File ${line} is not formated, please format it." - echo "1" > "${CHECK_RESULT_FILE}" - break - fi -done < "${CHECK_LIST_FILE}" + result=$(cat "${CHECK_RESULT_FILE}") + rm "${CHECK_RESULT_FILE}" + rm "${CHECK_LIST_FILE}" +popd -result=$(cat "${CHECK_RESULT_FILE}") -rm "${CHECK_RESULT_FILE}" -rm "${CHECK_LIST_FILE}" -cd "${CURRENT_PATH}" || exit 1 if [[ "X${result}" == "X0" ]]; then echo "Check PASS: specified files are well formated!" fi diff --git a/scripts/clean/ge_clean.sh b/scripts/clean/ge_clean.sh new file mode 100755 index 00000000..90fb939e --- /dev/null +++ b/scripts/clean/ge_clean.sh @@ -0,0 +1,90 @@ +#!/bin/bash +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +set -e + +PROJECT_HOME=${PROJECT_HOME:-$(dirname "$0")/../../} + +function help(){ + cat <<-EOF +Usage: ge clean [OPTIONS] + +Options: + -b, --build Clean build dir + -d, --docs Clean generate docs + -i, --install Clean dependenices + -a, --all Clean all + -h, --help +EOF + +} + +function clean_relative_dir(){ + rm -rf "${PROJECT_HOME}/${1:-output}" +} + +function parse_args(){ + parsed_args=$(getopt -a -o bdiah --long build,docs,install,all,help -- "$@") || { + help + exit 1 + } + + if [ $# -lt 1 ]; then + clean_relative_dir "build" + clean_relative_dir "output" + exit 1 + fi + + eval set -- "$parsed_args" + while true; do + case "$1" in + -b | --build) + clean_relative_dir "build" + clean_relative_dir "output" + ;; + -d | --docs) + clean_relative_dir "docs/doxygen" + ;; + -i | --install) + clean_relative_dir "deps" + ;; + -a | --all) + clean_relative_dir "deps" + clean_relative_dir "build" + clean_relative_dir "output" + clean_relative_dir "docs/doxygen" + ;; + -h | --help) + help + ;; + --) + shift; break; + ;; + *) + help; exit 1 + ;; + esac + shift + done +} + +function main(){ + parse_args "$@" +} + +main "$@" + +set +e \ No newline at end of file diff --git a/scripts/config/ge_config.sh b/scripts/config/ge_config.sh new file mode 100755 index 00000000..a5aba2a8 --- /dev/null +++ b/scripts/config/ge_config.sh @@ -0,0 +1,115 @@ +#!/bin/bash +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +set -e + +PROJECT_HOME=${PROJECT_HOME:-$(dirname "$0")/../../} +PROJECT_HOME=$(cd $PROJECT_HOME || return; pwd) + +function help(){ + cat <<-EOF +Usage: ge config [OPTIONS] + +update server config for ge, you need input all config info (ip, user, password) + +Options: + -i, --ip Config ip config + -u, --user Config user name + -p, --password Config password + -h, --help + +Example: ge config -i=121.36.**.** -u=asc**, -p=Asc***\#@\!\$ (Need add escape character \ before special charactor $、#、!) + +EOF + +} + +function write_config_file(){ + local IP=$1 + local USER=$2 + local PASSWORD=$3 + if [[ -z "$IP" ]] || [[ -z "$USER" ]] || [[ -z "$USER" ]]; then + echo "You need input all info （ip, user,password）obout server config !!!" + help + exit 1 + fi + + local PASSWORD=${PASSWORD//!/\\!} + local PASSWORD=${PASSWORD//#/\\#} + local PASSWORD=${PASSWORD/\$/\\\$} + local SERVER_CONFIG_FILE=${PROJECT_HOME}/scripts/config/server_config.sh + [ -n "${SERVER_CONFIG_FILE}" ] && rm -rf "${SERVER_CONFIG_FILE}" + + cat>${SERVER_CONFIG_FILE}<<-EOF +SERVER_PATH=http://${IP}/package/etrans +DEP_USER=${USER} +DEP_PASSWORD=${PASSWORD} + +EOF + +} + + + +function parse_args(){ + parsed_args=$(getopt -a -o i::u::p::h --long ip::,user::,password::,help -- "$@") || { + help + exit 1 + } + + if [ $# -lt 1 ]; then + help + exit 1 + fi + local IP= + local USER= + local PASSWORD= + + eval set -- "$parsed_args" + while true; do + case "$1" in + -i | --ip) + IP=$2 + ;; + -u | --user) + USER=$2 + ;; + -p | --password) + PASSWORD=$2 + ;; + -h | --help) + help; exit; + ;; + --) + shift; break; + ;; + *) + help; exit 1 + ;; + esac + shift 2 + done + + write_config_file $IP $USER $PASSWORD +} + +function main(){ + parse_args "$@" +} + +main "$@" + +set +e \ No newline at end of file diff --git a/scripts/cov/ge_cov.sh b/scripts/cov/ge_cov.sh new file mode 100755 index 00000000..27c5b5f7 --- /dev/null +++ b/scripts/cov/ge_cov.sh @@ -0,0 +1,136 @@ +#!/bin/bash +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +set -e + + +function help(){ + cat <<-EOF +Usage: ge cov [OPTIONS] + +Options: + -a, --all Full coverage + -i, --increment Increment coverage + -d, --directory Coverage of directory + -h, --help +EOF + +} + +PROJECT_HOME=${PROJECT_HOME:-$(dirname "$0")/../../} +PROJECT_HOME=$(cd $PROJECT_HOME || return; pwd) + +ALL_COV_GEN_PATH=${PROJECT_HOME}/cov/all +DIFF_FILE_PATH=${PROJECT_HOME}/cov/diff +DIFF_FILE_NAME=${DIFF_FILE_PATH}/inc_change_diff.txt + +function process_diff_format(){ + sed -i "s/--- a/--- \/code\/Turing\/graphEngine/g" ${DIFF_FILE_NAME} + sed -i "s/+++ b/+++ \/code\/Turing\/graphEngine/g" ${DIFF_FILE_NAME} +} + + +function add_cov_generate(){ + addlcov --diff ${ALL_COV_GEN_PATH}/coverage.info ${DIFF_FILE_NAME} -o ${PROJECT_HOME}/cov/diff/inc_coverage.info +} + +function gen_add_cov_html(){ + genhtml --prefix ${PROJECT_HOME} -o ${PROJECT_HOME}/cov/diff/html ${PROJECT_HOME}/cov/diff/inc_coverage.info --legend -t CHG --no-branch-coverage --no-function-coverage +} + +function increment_cov_for_directory(){ + [ -n "${DIFF_FILE_PATH}" ] && rm -rf "${DIFF_FILE_PATH}" + mkdir -p ${DIFF_FILE_PATH} + git diff HEAD -- $1 >>${DIFF_FILE_NAME} + process_diff_format + add_cov_generate + gen_add_cov_html +} + +function run_all_coverage(){ + [ -n "${ALL_COV_GEN_PATH}" ] && rm -rf ${ALL_COV_GEN_PATH} + mkdir -p ${ALL_COV_GEN_PATH} + pushd "${PWD}" >/dev/null + cd ${PROJECT_HOME} + lcov -c -d build/tests/ut/ge -d build/tests/ut/common/graph/ -o ${ALL_COV_GEN_PATH}/tmp.info + lcov -r ${ALL_COV_GEN_PATH}/tmp.info '*/output/*' '*/build/opensrc/*' '*/build/proto/*' '*/third_party/*' '*/tests/*' '/usr/local/*' '/usr/include/*' '*/metadef/*' '*/parser/*' -o ${ALL_COV_GEN_PATH}/coverage.info + cd ${ALL_COV_GEN_PATH} + genhtml coverage.info + popd >/dev/null +} + +function do_coverage_run(){ + local cov_mode=$1 + local directory_dir=$2 + + run_all_coverage + + if [ "$cov_mode" = "all" ]; then + exit 1 + elif [ -n "$directory_dir" ]; then + increment_cov_for_directory $directory_dir + else + increment_cov_for_directory "ge" + fi +} + +function parse_args(){ + parsed_args=$(getopt -a -o aid::h --long all,increment,directory::,help -- "$@") || { + help + exit 1 + } + + if [ $# -lt 1 ]; then + run_all_coverage + exit 1 + fi + + local cov_mode="increment" + local directory_dir= + eval set -- "$parsed_args" + while true; do + case "$1" in + -a | --all) + cov_mode="all" + ;; + -i | --increment) + ;; + -d | --directory) + directory_dir=$2 + shift + ;; + -h | --help) + help; exit 1; + ;; + --) + shift; break; + ;; + *) + help; exit 1; + ;; + esac + shift + done + do_coverage_run $cov_mode $directory_dir +} + +function main(){ + parse_args "$@" +} + +main "$@" + +set +e \ No newline at end of file diff --git a/scripts/docs/Doxyfile_all b/scripts/docs/Doxyfile_all new file mode 100755 index 00000000..100f4e62 --- /dev/null +++ b/scripts/docs/Doxyfile_all @@ -0,0 +1,2639 @@ +# Doxyfile 1.9.1 + +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + + +# This file describes the settings to be used by the documentation system +# doxygen (www.doxygen.org) for a project. +# +# All text after a double hash (##) is considered a comment and is placed in +# front of the TAG it is preceding. +# +# All text after a single hash (#) is considered a comment and will be ignored. +# The format is: +# TAG = value [value, ...] +# For lists, items can also be appended using: +# TAG += value [value, ...] +# Values that contain spaces should be placed between quotes (\" \"). + +#--------------------------------------------------------------------------- +# Project related configuration options +#--------------------------------------------------------------------------- + +# This tag specifies the encoding used for all characters in the configuration +# file that follow. The default is UTF-8 which is also the encoding used for all +# text before the first occurrence of this tag. Doxygen uses libiconv (or the +# iconv built into libc) for the transcoding. See +# https://www.gnu.org/software/libiconv/ for the list of possible encodings. +# The default value is: UTF-8. + +DOXYFILE_ENCODING = UTF-8 + +# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by +# double-quotes, unless you are using Doxywizard) that should identify the +# project for which the documentation is generated. This name is used in the +# title of most generated pages and in a few other places. +# The default value is: My Project. + +PROJECT_NAME = "graph engine" + +# The PROJECT_NUMBER tag can be used to enter a project or revision number. This +# could be handy for archiving the generated documentation or if some version +# control system is used. + +PROJECT_NUMBER = + +# Using the PROJECT_BRIEF tag one can provide an optional one line description +# for a project that appears at the top of each page and should give viewer a +# quick idea about the purpose of the project. Keep the description short. + +PROJECT_BRIEF = + +# With the PROJECT_LOGO tag one can specify a logo or an icon that is included +# in the documentation. The maximum height of the logo should not exceed 55 +# pixels and the maximum width should not exceed 200 pixels. Doxygen will copy +# the logo to the output directory. + +PROJECT_LOGO = + +# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path +# into which the generated documentation will be written. If a relative path is +# entered, it will be relative to the location where doxygen was started. If +# left blank the current directory will be used. + +OUTPUT_DIRECTORY = /code/Turing/graphEngine/docs/doxygen + +# If the CREATE_SUBDIRS tag is set to YES then doxygen will create 4096 sub- +# directories (in 2 levels) under the output directory of each output format and +# will distribute the generated files over these directories. Enabling this +# option can be useful when feeding doxygen a huge amount of source files, where +# putting all generated files in the same directory would otherwise causes +# performance problems for the file system. +# The default value is: NO. + +CREATE_SUBDIRS = YES + +# If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII +# characters to appear in the names of generated files. If set to NO, non-ASCII +# characters will be escaped, for example _xE3_x81_x84 will be used for Unicode +# U+3044. +# The default value is: NO. + +ALLOW_UNICODE_NAMES = NO + +# The OUTPUT_LANGUAGE tag is used to specify the language in which all +# documentation generated by doxygen is written. Doxygen will use this +# information to generate all constant output in the proper language. +# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Catalan, Chinese, +# Chinese-Traditional, Croatian, Czech, Danish, Dutch, English (United States), +# Esperanto, Farsi (Persian), Finnish, French, German, Greek, Hungarian, +# Indonesian, Italian, Japanese, Japanese-en (Japanese with English messages), +# Korean, Korean-en (Korean with English messages), Latvian, Lithuanian, +# Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, Romanian, Russian, +# Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, Swedish, Turkish, +# Ukrainian and Vietnamese. +# The default value is: English. + +OUTPUT_LANGUAGE = English + +# The OUTPUT_TEXT_DIRECTION tag is used to specify the direction in which all +# documentation generated by doxygen is written. Doxygen will use this +# information to generate all generated output in the proper direction. +# Possible values are: None, LTR, RTL and Context. +# The default value is: None. + +OUTPUT_TEXT_DIRECTION = None + +# If the BRIEF_MEMBER_DESC tag is set to YES, doxygen will include brief member +# descriptions after the members that are listed in the file and class +# documentation (similar to Javadoc). Set to NO to disable this. +# The default value is: YES. + +BRIEF_MEMBER_DESC = YES + +# If the REPEAT_BRIEF tag is set to YES, doxygen will prepend the brief +# description of a member or function before the detailed description +# +# Note: If both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the +# brief descriptions will be completely suppressed. +# The default value is: YES. + +REPEAT_BRIEF = YES + +# This tag implements a quasi-intelligent brief description abbreviator that is +# used to form the text in various listings. Each string in this list, if found +# as the leading text of the brief description, will be stripped from the text +# and the result, after processing the whole list, is used as the annotated +# text. Otherwise, the brief description is used as-is. If left blank, the +# following values are used ($name is automatically replaced with the name of +# the entity):The $name class, The $name widget, The $name file, is, provides, +# specifies, contains, represents, a, an and the. + +ABBREVIATE_BRIEF = "The $name class" \ + "The $name widget" \ + "The $name file" \ + is \ + provides \ + specifies \ + contains \ + represents \ + a \ + an \ + the + +# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then +# doxygen will generate a detailed section even if there is only a brief +# description. +# The default value is: NO. + +ALWAYS_DETAILED_SEC = NO + +# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all +# inherited members of a class in the documentation of that class as if those +# members were ordinary class members. Constructors, destructors and assignment +# operators of the base classes will not be shown. +# The default value is: NO. + +INLINE_INHERITED_MEMB = NO + +# If the FULL_PATH_NAMES tag is set to YES, doxygen will prepend the full path +# before files name in the file list and in the header files. If set to NO the +# shortest path that makes the file name unique will be used +# The default value is: YES. + +FULL_PATH_NAMES = YES + +# The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path. +# Stripping is only done if one of the specified strings matches the left-hand +# part of the path. The tag can be used to show relative paths in the file list. +# If left blank the directory from which doxygen is run is used as the path to +# strip. +# +# Note that you can specify absolute paths here, but also relative paths, which +# will be relative from the directory where doxygen is started. +# This tag requires that the tag FULL_PATH_NAMES is set to YES. + +STRIP_FROM_PATH = + +# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the +# path mentioned in the documentation of a class, which tells the reader which +# header file to include in order to use a class. If left blank only the name of +# the header file containing the class definition is used. Otherwise one should +# specify the list of include paths that are normally passed to the compiler +# using the -I flag. + +STRIP_FROM_INC_PATH = + +# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but +# less readable) file names. This can be useful is your file systems doesn't +# support long names like on DOS, Mac, or CD-ROM. +# The default value is: NO. + +SHORT_NAMES = NO + +# If the JAVADOC_AUTOBRIEF tag is set to YES then doxygen will interpret the +# first line (until the first dot) of a Javadoc-style comment as the brief +# description. If set to NO, the Javadoc-style will behave just like regular Qt- +# style comments (thus requiring an explicit @brief command for a brief +# description.) +# The default value is: NO. + +JAVADOC_AUTOBRIEF = NO + +# If the JAVADOC_BANNER tag is set to YES then doxygen will interpret a line +# such as +# /*************** +# as being the beginning of a Javadoc-style comment "banner". If set to NO, the +# Javadoc-style will behave just like regular comments and it will not be +# interpreted by doxygen. +# The default value is: NO. + +JAVADOC_BANNER = NO + +# If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first +# line (until the first dot) of a Qt-style comment as the brief description. If +# set to NO, the Qt-style will behave just like regular Qt-style comments (thus +# requiring an explicit \brief command for a brief description.) +# The default value is: NO. + +QT_AUTOBRIEF = NO + +# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make doxygen treat a +# multi-line C++ special comment block (i.e. a block of //! or /// comments) as +# a brief description. This used to be the default behavior. The new default is +# to treat a multi-line C++ comment block as a detailed description. Set this +# tag to YES if you prefer the old behavior instead. +# +# Note that setting this tag to YES also means that rational rose comments are +# not recognized any more. +# The default value is: NO. + +MULTILINE_CPP_IS_BRIEF = NO + +# By default Python docstrings are displayed as preformatted text and doxygen's +# special commands cannot be used. By setting PYTHON_DOCSTRING to NO the +# doxygen's special commands can be used and the contents of the docstring +# documentation blocks is shown as doxygen documentation. +# The default value is: YES. + +PYTHON_DOCSTRING = YES + +# If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the +# documentation from any documented member that it re-implements. +# The default value is: YES. + +INHERIT_DOCS = YES + +# If the SEPARATE_MEMBER_PAGES tag is set to YES then doxygen will produce a new +# page for each member. If set to NO, the documentation of a member will be part +# of the file/class/namespace that contains it. +# The default value is: NO. + +SEPARATE_MEMBER_PAGES = NO + +# The TAB_SIZE tag can be used to set the number of spaces in a tab. Doxygen +# uses this value to replace tabs by spaces in code fragments. +# Minimum value: 1, maximum value: 16, default value: 4. + +TAB_SIZE = 4 + +# This tag can be used to specify a number of aliases that act as commands in +# the documentation. An alias has the form: +# name=value +# For example adding +# "sideeffect=@par Side Effects:\n" +# will allow you to put the command \sideeffect (or @sideeffect) in the +# documentation, which will result in a user-defined paragraph with heading +# "Side Effects:". You can put \n's in the value part of an alias to insert +# newlines (in the resulting output). You can put ^^ in the value part of an +# alias to insert a newline as if a physical newline was in the original file. +# When you need a literal { or } or , in the value part of an alias you have to +# escape them by means of a backslash (\), this can lead to conflicts with the +# commands \{ and \} for these it is advised to use the version @{ and @} or use +# a double escape (\\{ and \\}) + +ALIASES = + +# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources +# only. Doxygen will then generate output that is more tailored for C. For +# instance, some of the names that are used will be different. The list of all +# members will be omitted, etc. +# The default value is: NO. + +OPTIMIZE_OUTPUT_FOR_C = NO + +# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java or +# Python sources only. Doxygen will then generate output that is more tailored +# for that language. For instance, namespaces will be presented as packages, +# qualified scopes will look different, etc. +# The default value is: NO. + +OPTIMIZE_OUTPUT_JAVA = NO + +# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran +# sources. Doxygen will then generate output that is tailored for Fortran. +# The default value is: NO. + +OPTIMIZE_FOR_FORTRAN = NO + +# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL +# sources. Doxygen will then generate output that is tailored for VHDL. +# The default value is: NO. + +OPTIMIZE_OUTPUT_VHDL = NO + +# Set the OPTIMIZE_OUTPUT_SLICE tag to YES if your project consists of Slice +# sources only. Doxygen will then generate output that is more tailored for that +# language. For instance, namespaces will be presented as modules, types will be +# separated into more groups, etc. +# The default value is: NO. + +OPTIMIZE_OUTPUT_SLICE = NO + +# Doxygen selects the parser to use depending on the extension of the files it +# parses. With this tag you can assign which parser to use for a given +# extension. Doxygen has a built-in mapping, but you can override or extend it +# using this tag. The format is ext=language, where ext is a file extension, and +# language is one of the parsers supported by doxygen: IDL, Java, JavaScript, +# Csharp (C#), C, C++, D, PHP, md (Markdown), Objective-C, Python, Slice, VHDL, +# Fortran (fixed format Fortran: FortranFixed, free formatted Fortran: +# FortranFree, unknown formatted Fortran: Fortran. In the later case the parser +# tries to guess whether the code is fixed or free formatted code, this is the +# default for Fortran type files). For instance to make doxygen treat .inc files +# as Fortran files (default is PHP), and .f files as C (default is Fortran), +# use: inc=Fortran f=C. +# +# Note: For files without extension you can use no_extension as a placeholder. +# +# Note that for custom extensions you also need to set FILE_PATTERNS otherwise +# the files are not read by doxygen. When specifying no_extension you should add +# * to the FILE_PATTERNS. +# +# Note see also the list of default file extension mappings. + +EXTENSION_MAPPING = + +# If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments +# according to the Markdown format, which allows for more readable +# documentation. See https://daringfireball.net/projects/markdown/ for details. +# The output of markdown processing is further processed by doxygen, so you can +# mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in +# case of backward compatibilities issues. +# The default value is: YES. + +MARKDOWN_SUPPORT = YES + +# When the TOC_INCLUDE_HEADINGS tag is set to a non-zero value, all headings up +# to that level are automatically included in the table of contents, even if +# they do not have an id attribute. +# Note: This feature currently applies only to Markdown headings. +# Minimum value: 0, maximum value: 99, default value: 5. +# This tag requires that the tag MARKDOWN_SUPPORT is set to YES. + +TOC_INCLUDE_HEADINGS = 5 + +# When enabled doxygen tries to link words that correspond to documented +# classes, or namespaces to their corresponding documentation. Such a link can +# be prevented in individual cases by putting a % sign in front of the word or +# globally by setting AUTOLINK_SUPPORT to NO. +# The default value is: YES. + +AUTOLINK_SUPPORT = YES + +# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want +# to include (a tag file for) the STL sources as input, then you should set this +# tag to YES in order to let doxygen match functions declarations and +# definitions whose arguments contain STL classes (e.g. func(std::string); +# versus func(std::string) {}). This also make the inheritance and collaboration +# diagrams that involve STL classes more complete and accurate. +# The default value is: NO. + +BUILTIN_STL_SUPPORT = NO + +# If you use Microsoft's C++/CLI language, you should set this option to YES to +# enable parsing support. +# The default value is: NO. + +CPP_CLI_SUPPORT = NO + +# Set the SIP_SUPPORT tag to YES if your project consists of sip (see: +# https://www.riverbankcomputing.com/software/sip/intro) sources only. Doxygen +# will parse them like normal C++ but will assume all classes use public instead +# of private inheritance when no explicit protection keyword is present. +# The default value is: NO. + +SIP_SUPPORT = NO + +# For Microsoft's IDL there are propget and propput attributes to indicate +# getter and setter methods for a property. Setting this option to YES will make +# doxygen to replace the get and set methods by a property in the documentation. +# This will only work if the methods are indeed getting or setting a simple +# type. If this is not the case, or you want to show the methods anyway, you +# should set this option to NO. +# The default value is: YES. + +IDL_PROPERTY_SUPPORT = YES + +# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC +# tag is set to YES then doxygen will reuse the documentation of the first +# member in the group (if any) for the other members of the group. By default +# all members of a group must be documented explicitly. +# The default value is: NO. + +DISTRIBUTE_GROUP_DOC = NO + +# If one adds a struct or class to a group and this option is enabled, then also +# any nested class or struct is added to the same group. By default this option +# is disabled and one has to add nested compounds explicitly via \ingroup. +# The default value is: NO. + +GROUP_NESTED_COMPOUNDS = NO + +# Set the SUBGROUPING tag to YES to allow class member groups of the same type +# (for instance a group of public functions) to be put as a subgroup of that +# type (e.g. under the Public Functions section). Set it to NO to prevent +# subgrouping. Alternatively, this can be done per class using the +# \nosubgrouping command. +# The default value is: YES. + +SUBGROUPING = YES + +# When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and unions +# are shown inside the group in which they are included (e.g. using \ingroup) +# instead of on a separate page (for HTML and Man pages) or section (for LaTeX +# and RTF). +# +# Note that this feature does not work in combination with +# SEPARATE_MEMBER_PAGES. +# The default value is: NO. + +INLINE_GROUPED_CLASSES = NO + +# When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and unions +# with only public data fields or simple typedef fields will be shown inline in +# the documentation of the scope in which they are defined (i.e. file, +# namespace, or group documentation), provided this scope is documented. If set +# to NO, structs, classes, and unions are shown on a separate page (for HTML and +# Man pages) or section (for LaTeX and RTF). +# The default value is: NO. + +INLINE_SIMPLE_STRUCTS = NO + +# When TYPEDEF_HIDES_STRUCT tag is enabled, a typedef of a struct, union, or +# enum is documented as struct, union, or enum with the name of the typedef. So +# typedef struct TypeS {} TypeT, will appear in the documentation as a struct +# with name TypeT. When disabled the typedef will appear as a member of a file, +# namespace, or class. And the struct will be named TypeS. This can typically be +# useful for C code in case the coding convention dictates that all compound +# types are typedef'ed and only the typedef is referenced, never the tag name. +# The default value is: NO. + +TYPEDEF_HIDES_STRUCT = YES + +# The size of the symbol lookup cache can be set using LOOKUP_CACHE_SIZE. This +# cache is used to resolve symbols given their name and scope. Since this can be +# an expensive process and often the same symbol appears multiple times in the +# code, doxygen keeps a cache of pre-resolved symbols. If the cache is too small +# doxygen will become slower. If the cache is too large, memory is wasted. The +# cache size is given by this formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range +# is 0..9, the default is 0, corresponding to a cache size of 2^16=65536 +# symbols. At the end of a run doxygen will report the cache usage and suggest +# the optimal cache size from a speed point of view. +# Minimum value: 0, maximum value: 9, default value: 0. + +LOOKUP_CACHE_SIZE = 0 + +# The NUM_PROC_THREADS specifies the number threads doxygen is allowed to use +# during processing. When set to 0 doxygen will based this on the number of +# cores available in the system. You can set it explicitly to a value larger +# than 0 to get more control over the balance between CPU load and processing +# speed. At this moment only the input processing can be done using multiple +# threads. Since this is still an experimental feature the default is set to 1, +# which efficively disables parallel processing. Please report any issues you +# encounter. Generating dot graphs in parallel is controlled by the +# DOT_NUM_THREADS setting. +# Minimum value: 0, maximum value: 32, default value: 1. + +NUM_PROC_THREADS = 1 + +#--------------------------------------------------------------------------- +# Build related configuration options +#--------------------------------------------------------------------------- + +# If the EXTRACT_ALL tag is set to YES, doxygen will assume all entities in +# documentation are documented, even if no documentation was available. Private +# class members and static file members will be hidden unless the +# EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES. +# Note: This will also disable the warnings about undocumented members that are +# normally produced when WARNINGS is set to YES. +# The default value is: NO. + +EXTRACT_ALL = YES + +# If the EXTRACT_PRIVATE tag is set to YES, all private members of a class will +# be included in the documentation. +# The default value is: NO. + +EXTRACT_PRIVATE = NO + +# If the EXTRACT_PRIV_VIRTUAL tag is set to YES, documented private virtual +# methods of a class will be included in the documentation. +# The default value is: NO. + +EXTRACT_PRIV_VIRTUAL = NO + +# If the EXTRACT_PACKAGE tag is set to YES, all members with package or internal +# scope will be included in the documentation. +# The default value is: NO. + +EXTRACT_PACKAGE = NO + +# If the EXTRACT_STATIC tag is set to YES, all static members of a file will be +# included in the documentation. +# The default value is: NO. + +EXTRACT_STATIC = NO + +# If the EXTRACT_LOCAL_CLASSES tag is set to YES, classes (and structs) defined +# locally in source files will be included in the documentation. If set to NO, +# only classes defined in header files are included. Does not have any effect +# for Java sources. +# The default value is: YES. + +EXTRACT_LOCAL_CLASSES = YES + +# This flag is only useful for Objective-C code. If set to YES, local methods, +# which are defined in the implementation section but not in the interface are +# included in the documentation. If set to NO, only methods in the interface are +# included. +# The default value is: NO. + +EXTRACT_LOCAL_METHODS = NO + +# If this flag is set to YES, the members of anonymous namespaces will be +# extracted and appear in the documentation as a namespace called +# 'anonymous_namespace{file}', where file will be replaced with the base name of +# the file that contains the anonymous namespace. By default anonymous namespace +# are hidden. +# The default value is: NO. + +EXTRACT_ANON_NSPACES = NO + +# If this flag is set to YES, the name of an unnamed parameter in a declaration +# will be determined by the corresponding definition. By default unnamed +# parameters remain unnamed in the output. +# The default value is: YES. + +RESOLVE_UNNAMED_PARAMS = YES + +# If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all +# undocumented members inside documented classes or files. If set to NO these +# members will be included in the various overviews, but no documentation +# section is generated. This option has no effect if EXTRACT_ALL is enabled. +# The default value is: NO. + +HIDE_UNDOC_MEMBERS = NO + +# If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all +# undocumented classes that are normally visible in the class hierarchy. If set +# to NO, these classes will be included in the various overviews. This option +# has no effect if EXTRACT_ALL is enabled. +# The default value is: NO. + +HIDE_UNDOC_CLASSES = NO + +# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend +# declarations. If set to NO, these declarations will be included in the +# documentation. +# The default value is: NO. + +HIDE_FRIEND_COMPOUNDS = NO + +# If the HIDE_IN_BODY_DOCS tag is set to YES, doxygen will hide any +# documentation blocks found inside the body of a function. If set to NO, these +# blocks will be appended to the function's detailed documentation block. +# The default value is: NO. + +HIDE_IN_BODY_DOCS = NO + +# The INTERNAL_DOCS tag determines if documentation that is typed after a +# \internal command is included. If the tag is set to NO then the documentation +# will be excluded. Set it to YES to include the internal documentation. +# The default value is: NO. + +INTERNAL_DOCS = NO + +# With the correct setting of option CASE_SENSE_NAMES doxygen will better be +# able to match the capabilities of the underlying filesystem. In case the +# filesystem is case sensitive (i.e. it supports files in the same directory +# whose names only differ in casing), the option must be set to YES to properly +# deal with such files in case they appear in the input. For filesystems that +# are not case sensitive the option should be be set to NO to properly deal with +# output files written for symbols that only differ in casing, such as for two +# classes, one named CLASS and the other named Class, and to also support +# references to files without having to specify the exact matching casing. On +# Windows (including Cygwin) and MacOS, users should typically set this option +# to NO, whereas on Linux or other Unix flavors it should typically be set to +# YES. +# The default value is: system dependent. + +CASE_SENSE_NAMES = NO + +# If the HIDE_SCOPE_NAMES tag is set to NO then doxygen will show members with +# their full class and namespace scopes in the documentation. If set to YES, the +# scope will be hidden. +# The default value is: NO. + +HIDE_SCOPE_NAMES = NO + +# If the HIDE_COMPOUND_REFERENCE tag is set to NO (default) then doxygen will +# append additional text to a page's title, such as Class Reference. If set to +# YES the compound reference will be hidden. +# The default value is: NO. + +HIDE_COMPOUND_REFERENCE= NO + +# If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of +# the files that are included by a file in the documentation of that file. +# The default value is: YES. + +SHOW_INCLUDE_FILES = YES + +# If the SHOW_GROUPED_MEMB_INC tag is set to YES then Doxygen will add for each +# grouped member an include statement to the documentation, telling the reader +# which file to include in order to use the member. +# The default value is: NO. + +SHOW_GROUPED_MEMB_INC = NO + +# If the FORCE_LOCAL_INCLUDES tag is set to YES then doxygen will list include +# files with double quotes in the documentation rather than with sharp brackets. +# The default value is: NO. + +FORCE_LOCAL_INCLUDES = NO + +# If the INLINE_INFO tag is set to YES then a tag [inline] is inserted in the +# documentation for inline members. +# The default value is: YES. + +INLINE_INFO = YES + +# If the SORT_MEMBER_DOCS tag is set to YES then doxygen will sort the +# (detailed) documentation of file and class members alphabetically by member +# name. If set to NO, the members will appear in declaration order. +# The default value is: YES. + +SORT_MEMBER_DOCS = YES + +# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the brief +# descriptions of file, namespace and class members alphabetically by member +# name. If set to NO, the members will appear in declaration order. Note that +# this will also influence the order of the classes in the class list. +# The default value is: NO. + +SORT_BRIEF_DOCS = NO + +# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen will sort the +# (brief and detailed) documentation of class members so that constructors and +# destructors are listed first. If set to NO the constructors will appear in the +# respective orders defined by SORT_BRIEF_DOCS and SORT_MEMBER_DOCS. +# Note: If SORT_BRIEF_DOCS is set to NO this option is ignored for sorting brief +# member documentation. +# Note: If SORT_MEMBER_DOCS is set to NO this option is ignored for sorting +# detailed member documentation. +# The default value is: NO. + +SORT_MEMBERS_CTORS_1ST = NO + +# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the hierarchy +# of group names into alphabetical order. If set to NO the group names will +# appear in their defined order. +# The default value is: NO. + +SORT_GROUP_NAMES = NO + +# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by +# fully-qualified names, including namespaces. If set to NO, the class list will +# be sorted only by class name, not including the namespace part. +# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. +# Note: This option applies only to the class list, not to the alphabetical +# list. +# The default value is: NO. + +SORT_BY_SCOPE_NAME = NO + +# If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to do proper +# type resolution of all parameters of a function it will reject a match between +# the prototype and the implementation of a member function even if there is +# only one candidate or it is obvious which candidate to choose by doing a +# simple string match. By disabling STRICT_PROTO_MATCHING doxygen will still +# accept a match between prototype and implementation in such cases. +# The default value is: NO. + +STRICT_PROTO_MATCHING = NO + +# The GENERATE_TODOLIST tag can be used to enable (YES) or disable (NO) the todo +# list. This list is created by putting \todo commands in the documentation. +# The default value is: YES. + +GENERATE_TODOLIST = YES + +# The GENERATE_TESTLIST tag can be used to enable (YES) or disable (NO) the test +# list. This list is created by putting \test commands in the documentation. +# The default value is: YES. + +GENERATE_TESTLIST = YES + +# The GENERATE_BUGLIST tag can be used to enable (YES) or disable (NO) the bug +# list. This list is created by putting \bug commands in the documentation. +# The default value is: YES. + +GENERATE_BUGLIST = YES + +# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or disable (NO) +# the deprecated list. This list is created by putting \deprecated commands in +# the documentation. +# The default value is: YES. + +GENERATE_DEPRECATEDLIST= YES + +# The ENABLED_SECTIONS tag can be used to enable conditional documentation +# sections, marked by \if ... \endif and \cond +# ... \endcond blocks. + +ENABLED_SECTIONS = + +# The MAX_INITIALIZER_LINES tag determines the maximum number of lines that the +# initial value of a variable or macro / define can have for it to appear in the +# documentation. If the initializer consists of more lines than specified here +# it will be hidden. Use a value of 0 to hide initializers completely. The +# appearance of the value of individual variables and macros / defines can be +# controlled using \showinitializer or \hideinitializer command in the +# documentation regardless of this setting. +# Minimum value: 0, maximum value: 10000, default value: 30. + +MAX_INITIALIZER_LINES = 30 + +# Set the SHOW_USED_FILES tag to NO to disable the list of files generated at +# the bottom of the documentation of classes and structs. If set to YES, the +# list will mention the files that were used to generate the documentation. +# The default value is: YES. + +SHOW_USED_FILES = YES + +# Set the SHOW_FILES tag to NO to disable the generation of the Files page. This +# will remove the Files entry from the Quick Index and from the Folder Tree View +# (if specified). +# The default value is: YES. + +SHOW_FILES = YES + +# Set the SHOW_NAMESPACES tag to NO to disable the generation of the Namespaces +# page. This will remove the Namespaces entry from the Quick Index and from the +# Folder Tree View (if specified). +# The default value is: YES. + +SHOW_NAMESPACES = YES + +# The FILE_VERSION_FILTER tag can be used to specify a program or script that +# doxygen should invoke to get the current version for each file (typically from +# the version control system). Doxygen will invoke the program by executing (via +# popen()) the command command input-file, where command is the value of the +# FILE_VERSION_FILTER tag, and input-file is the name of an input file provided +# by doxygen. Whatever the program writes to standard output is used as the file +# version. For an example see the documentation. + +FILE_VERSION_FILTER = + +# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed +# by doxygen. The layout file controls the global structure of the generated +# output files in an output format independent way. To create the layout file +# that represents doxygen's defaults, run doxygen with the -l option. You can +# optionally specify a file name after the option, if omitted DoxygenLayout.xml +# will be used as the name of the layout file. +# +# Note that if you run doxygen from a directory containing a file called +# DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE +# tag is left empty. + +LAYOUT_FILE = + +# The CITE_BIB_FILES tag can be used to specify one or more bib files containing +# the reference definitions. This must be a list of .bib files. The .bib +# extension is automatically appended if omitted. This requires the bibtex tool +# to be installed. See also https://en.wikipedia.org/wiki/BibTeX for more info. +# For LaTeX the style of the bibliography can be controlled using +# LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the +# search path. See also \cite for info how to create references. + +CITE_BIB_FILES = + +#--------------------------------------------------------------------------- +# Configuration options related to warning and progress messages +#--------------------------------------------------------------------------- + +# The QUIET tag can be used to turn on/off the messages that are generated to +# standard output by doxygen. If QUIET is set to YES this implies that the +# messages are off. +# The default value is: NO. + +QUIET = YES + +# The WARNINGS tag can be used to turn on/off the warning messages that are +# generated to standard error (stderr) by doxygen. If WARNINGS is set to YES +# this implies that the warnings are on. +# +# Tip: Turn warnings on while writing the documentation. +# The default value is: YES. + +WARNINGS = YES + +# If the WARN_IF_UNDOCUMENTED tag is set to YES then doxygen will generate +# warnings for undocumented members. If EXTRACT_ALL is set to YES then this flag +# will automatically be disabled. +# The default value is: YES. + +WARN_IF_UNDOCUMENTED = YES + +# If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for +# potential errors in the documentation, such as not documenting some parameters +# in a documented function, or documenting parameters that don't exist or using +# markup commands wrongly. +# The default value is: YES. + +WARN_IF_DOC_ERROR = YES + +# This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that +# are documented, but have no documentation for their parameters or return +# value. If set to NO, doxygen will only warn about wrong or incomplete +# parameter documentation, but not about the absence of documentation. If +# EXTRACT_ALL is set to YES then this flag will automatically be disabled. +# The default value is: NO. + +WARN_NO_PARAMDOC = NO + +# If the WARN_AS_ERROR tag is set to YES then doxygen will immediately stop when +# a warning is encountered. If the WARN_AS_ERROR tag is set to FAIL_ON_WARNINGS +# then doxygen will continue running as if WARN_AS_ERROR tag is set to NO, but +# at the end of the doxygen process doxygen will return with a non-zero status. +# Possible values are: NO, YES and FAIL_ON_WARNINGS. +# The default value is: NO. + +WARN_AS_ERROR = NO + +# The WARN_FORMAT tag determines the format of the warning messages that doxygen +# can produce. The string should contain the $file, $line, and $text tags, which +# will be replaced by the file and line number from which the warning originated +# and the warning text. Optionally the format may contain $version, which will +# be replaced by the version of the file (if it could be obtained via +# FILE_VERSION_FILTER) +# The default value is: $file:$line: $text. + +WARN_FORMAT = "$file:$line: $text" + +# The WARN_LOGFILE tag can be used to specify a file to which warning and error +# messages should be written. If left blank the output is written to standard +# error (stderr). + +WARN_LOGFILE = + +#--------------------------------------------------------------------------- +# Configuration options related to the input files +#--------------------------------------------------------------------------- + +# The INPUT tag is used to specify the files and/or directories that contain +# documented source files. You may enter file names like myfile.cpp or +# directories like /usr/src/myproject. Separate the files or directories with +# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING +# Note: If this tag is empty the current directory is searched. + +INPUT =/code/Turing/graphEngine/ge \ + /code/Turing/graphEngine/inc \ + /code/Turing/graphEngine/parser \ + /code/Turing/graphEngine/metadef \ + /code/Turing/graphEngine/third_party/inc + + +# This tag can be used to specify the character encoding of the source files +# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses +# libiconv (or the iconv built into libc) for the transcoding. See the libiconv +# documentation (see: +# https://www.gnu.org/software/libiconv/) for the list of possible encodings. +# The default value is: UTF-8. + +INPUT_ENCODING = UTF-8 + +# If the value of the INPUT tag contains directories, you can use the +# FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and +# *.h) to filter out the source-files in the directories. +# +# Note that for custom extensions or not directly supported extensions you also +# need to set EXTENSION_MAPPING for the extension otherwise the files are not +# read by doxygen. +# +# Note the list of default checked file patterns might differ from the list of +# default file extension mappings. +# +# If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp, +# *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, +# *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc, +# *.m, *.markdown, *.md, *.mm, *.dox (to be provided as doxygen C comment), +# *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, *.f18, *.f, *.for, *.vhd, *.vhdl, +# *.ucf, *.qsf and *.ice. + +FILE_PATTERNS = *.c \ + *.cc \ + *.cxx \ + *.cpp \ + *.c++ \ + *.java \ + *.ii \ + *.ixx \ + *.ipp \ + *.i++ \ + *.inl \ + *.idl \ + *.ddl \ + *.odl \ + *.h \ + *.hh \ + *.hxx \ + *.hpp \ + *.h++ \ + *.cs \ + *.d \ + *.php \ + *.php4 \ + *.php5 \ + *.phtml \ + *.inc \ + *.m \ + *.markdown \ + *.md \ + *.mm \ + *.dox \ + *.py \ + *.pyw \ + *.f90 \ + *.f95 \ + *.f03 \ + *.f08 \ + *.f18 \ + *.f \ + *.for \ + *.vhd \ + *.vhdl \ + *.ucf \ + *.qsf \ + *.ice + +# The RECURSIVE tag can be used to specify whether or not subdirectories should +# be searched for input files as well. +# The default value is: NO. + +RECURSIVE = YES + +# The EXCLUDE tag can be used to specify files and/or directories that should be +# excluded from the INPUT source files. This way you can easily exclude a +# subdirectory from a directory tree whose root is specified with the INPUT tag. +# +# Note that relative paths are relative to the directory from which doxygen is +# run. + +EXCLUDE = + +# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or +# directories that are symbolic links (a Unix file system feature) are excluded +# from the input. +# The default value is: NO. + +EXCLUDE_SYMLINKS = NO + +# If the value of the INPUT tag contains directories, you can use the +# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude +# certain files from those directories. +# +# Note that the wildcards are matched against the file with absolute path, so to +# exclude all test directories for example use the pattern */test/* + +EXCLUDE_PATTERNS = + +# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names +# (namespaces, classes, functions, etc.) that should be excluded from the +# output. The symbol name can be a fully qualified name, a word, or if the +# wildcard * is used, a substring. Examples: ANamespace, AClass, +# AClass::ANamespace, ANamespace::*Test +# +# Note that the wildcards are matched against the file with absolute path, so to +# exclude all test directories use the pattern */test/* + +EXCLUDE_SYMBOLS = + +# The EXAMPLE_PATH tag can be used to specify one or more files or directories +# that contain example code fragments that are included (see the \include +# command). + +EXAMPLE_PATH = + +# If the value of the EXAMPLE_PATH tag contains directories, you can use the +# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and +# *.h) to filter out the source-files in the directories. If left blank all +# files are included. + +EXAMPLE_PATTERNS = * + +# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be +# searched for input files to be used with the \include or \dontinclude commands +# irrespective of the value of the RECURSIVE tag. +# The default value is: NO. + +EXAMPLE_RECURSIVE = NO + +# The IMAGE_PATH tag can be used to specify one or more files or directories +# that contain images that are to be included in the documentation (see the +# \image command). + +IMAGE_PATH = + +# The INPUT_FILTER tag can be used to specify a program that doxygen should +# invoke to filter for each input file. Doxygen will invoke the filter program +# by executing (via popen()) the command: +# +# +# +# where is the value of the INPUT_FILTER tag, and is the +# name of an input file. Doxygen will then use the output that the filter +# program writes to standard output. If FILTER_PATTERNS is specified, this tag +# will be ignored. +# +# Note that the filter must not add or remove lines; it is applied before the +# code is scanned, but not when the output code is generated. If lines are added +# or removed, the anchors will not be placed correctly. +# +# Note that for custom extensions or not directly supported extensions you also +# need to set EXTENSION_MAPPING for the extension otherwise the files are not +# properly processed by doxygen. + +INPUT_FILTER = + +# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern +# basis. Doxygen will compare the file name with each pattern and apply the +# filter if there is a match. The filters are a list of the form: pattern=filter +# (like *.cpp=my_cpp_filter). See INPUT_FILTER for further information on how +# filters are used. If the FILTER_PATTERNS tag is empty or if none of the +# patterns match the file name, INPUT_FILTER is applied. +# +# Note that for custom extensions or not directly supported extensions you also +# need to set EXTENSION_MAPPING for the extension otherwise the files are not +# properly processed by doxygen. + +FILTER_PATTERNS = + +# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using +# INPUT_FILTER) will also be used to filter the input files that are used for +# producing the source files to browse (i.e. when SOURCE_BROWSER is set to YES). +# The default value is: NO. + +FILTER_SOURCE_FILES = NO + +# The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file +# pattern. A pattern will override the setting for FILTER_PATTERN (if any) and +# it is also possible to disable source filtering for a specific pattern using +# *.ext= (so without naming a filter). +# This tag requires that the tag FILTER_SOURCE_FILES is set to YES. + +FILTER_SOURCE_PATTERNS = + +# If the USE_MDFILE_AS_MAINPAGE tag refers to the name of a markdown file that +# is part of the input, its contents will be placed on the main page +# (index.html). This can be useful if you have a project on for instance GitHub +# and want to reuse the introduction page also for the doxygen output. + +USE_MDFILE_AS_MAINPAGE = + +#--------------------------------------------------------------------------- +# Configuration options related to source browsing +#--------------------------------------------------------------------------- + +# If the SOURCE_BROWSER tag is set to YES then a list of source files will be +# generated. Documented entities will be cross-referenced with these sources. +# +# Note: To get rid of all source code in the generated output, make sure that +# also VERBATIM_HEADERS is set to NO. +# The default value is: NO. + +SOURCE_BROWSER = NO + +# Setting the INLINE_SOURCES tag to YES will include the body of functions, +# classes and enums directly into the documentation. +# The default value is: NO. + +INLINE_SOURCES = NO + +# Setting the STRIP_CODE_COMMENTS tag to YES will instruct doxygen to hide any +# special comment blocks from generated source code fragments. Normal C, C++ and +# Fortran comments will always remain visible. +# The default value is: YES. + +STRIP_CODE_COMMENTS = YES + +# If the REFERENCED_BY_RELATION tag is set to YES then for each documented +# entity all documented functions referencing it will be listed. +# The default value is: NO. + +REFERENCED_BY_RELATION = YES + +# If the REFERENCES_RELATION tag is set to YES then for each documented function +# all documented entities called/used by that function will be listed. +# The default value is: NO. + +REFERENCES_RELATION = YES + +# If the REFERENCES_LINK_SOURCE tag is set to YES and SOURCE_BROWSER tag is set +# to YES then the hyperlinks from functions in REFERENCES_RELATION and +# REFERENCED_BY_RELATION lists will link to the source code. Otherwise they will +# link to the documentation. +# The default value is: YES. + +REFERENCES_LINK_SOURCE = YES + +# If SOURCE_TOOLTIPS is enabled (the default) then hovering a hyperlink in the +# source code will show a tooltip with additional information such as prototype, +# brief description and links to the definition and documentation. Since this +# will make the HTML file larger and loading of large files a bit slower, you +# can opt to disable this feature. +# The default value is: YES. +# This tag requires that the tag SOURCE_BROWSER is set to YES. + +SOURCE_TOOLTIPS = YES + +# If the USE_HTAGS tag is set to YES then the references to source code will +# point to the HTML generated by the htags(1) tool instead of doxygen built-in +# source browser. The htags tool is part of GNU's global source tagging system +# (see https://www.gnu.org/software/global/global.html). You will need version +# 4.8.6 or higher. +# +# To use it do the following: +# - Install the latest version of global +# - Enable SOURCE_BROWSER and USE_HTAGS in the configuration file +# - Make sure the INPUT points to the root of the source tree +# - Run doxygen as normal +# +# Doxygen will invoke htags (and that will in turn invoke gtags), so these +# tools must be available from the command line (i.e. in the search path). +# +# The result: instead of the source browser generated by doxygen, the links to +# source code will now point to the output of htags. +# The default value is: NO. +# This tag requires that the tag SOURCE_BROWSER is set to YES. + +USE_HTAGS = NO + +# If the VERBATIM_HEADERS tag is set the YES then doxygen will generate a +# verbatim copy of the header file for each class for which an include is +# specified. Set to NO to disable this. +# See also: Section \class. +# The default value is: YES. + +VERBATIM_HEADERS = YES + +#--------------------------------------------------------------------------- +# Configuration options related to the alphabetical class index +#--------------------------------------------------------------------------- + +# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index of all +# compounds will be generated. Enable this if the project contains a lot of +# classes, structs, unions or interfaces. +# The default value is: YES. + +ALPHABETICAL_INDEX = YES + +# In case all classes in a project start with a common prefix, all classes will +# be put under the same header in the alphabetical index. The IGNORE_PREFIX tag +# can be used to specify a prefix (or a list of prefixes) that should be ignored +# while generating the index headers. +# This tag requires that the tag ALPHABETICAL_INDEX is set to YES. + +IGNORE_PREFIX = + +#--------------------------------------------------------------------------- +# Configuration options related to the HTML output +#--------------------------------------------------------------------------- + +# If the GENERATE_HTML tag is set to YES, doxygen will generate HTML output +# The default value is: YES. + +GENERATE_HTML = YES + +# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a +# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of +# it. +# The default directory is: html. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_OUTPUT = html + +# The HTML_FILE_EXTENSION tag can be used to specify the file extension for each +# generated HTML page (for example: .htm, .php, .asp). +# The default value is: .html. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_FILE_EXTENSION = .html + +# The HTML_HEADER tag can be used to specify a user-defined HTML header file for +# each generated HTML page. If the tag is left blank doxygen will generate a +# standard header. +# +# To get valid HTML the header file that includes any scripts and style sheets +# that doxygen needs, which is dependent on the configuration options used (e.g. +# the setting GENERATE_TREEVIEW). It is highly recommended to start with a +# default header using +# doxygen -w html new_header.html new_footer.html new_stylesheet.css +# YourConfigFile +# and then modify the file new_header.html. See also section "Doxygen usage" +# for information on how to generate the default header that doxygen normally +# uses. +# Note: The header is subject to change so you typically have to regenerate the +# default header when upgrading to a newer version of doxygen. For a description +# of the possible markers and block names see the documentation. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_HEADER = + +# The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each +# generated HTML page. If the tag is left blank doxygen will generate a standard +# footer. See HTML_HEADER for more information on how to generate a default +# footer and what special commands can be used inside the footer. See also +# section "Doxygen usage" for information on how to generate the default footer +# that doxygen normally uses. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_FOOTER = + +# The HTML_STYLESHEET tag can be used to specify a user-defined cascading style +# sheet that is used by each HTML page. It can be used to fine-tune the look of +# the HTML output. If left blank doxygen will generate a default style sheet. +# See also section "Doxygen usage" for information on how to generate the style +# sheet that doxygen normally uses. +# Note: It is recommended to use HTML_EXTRA_STYLESHEET instead of this tag, as +# it is more robust and this tag (HTML_STYLESHEET) will in the future become +# obsolete. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_STYLESHEET = + +# The HTML_EXTRA_STYLESHEET tag can be used to specify additional user-defined +# cascading style sheets that are included after the standard style sheets +# created by doxygen. Using this option one can overrule certain style aspects. +# This is preferred over using HTML_STYLESHEET since it does not replace the +# standard style sheet and is therefore more robust against future updates. +# Doxygen will copy the style sheet files to the output directory. +# Note: The order of the extra style sheet files is of importance (e.g. the last +# style sheet in the list overrules the setting of the previous ones in the +# list). For an example see the documentation. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_EXTRA_STYLESHEET = + +# The HTML_EXTRA_FILES tag can be used to specify one or more extra images or +# other source files which should be copied to the HTML output directory. Note +# that these files will be copied to the base HTML output directory. Use the +# $relpath^ marker in the HTML_HEADER and/or HTML_FOOTER files to load these +# files. In the HTML_STYLESHEET file, use the file name only. Also note that the +# files will be copied as-is; there are no commands or markers available. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_EXTRA_FILES = + +# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen +# will adjust the colors in the style sheet and background images according to +# this color. Hue is specified as an angle on a colorwheel, see +# https://en.wikipedia.org/wiki/Hue for more information. For instance the value +# 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300 +# purple, and 360 is red again. +# Minimum value: 0, maximum value: 359, default value: 220. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_COLORSTYLE_HUE = 220 + +# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors +# in the HTML output. For a value of 0 the output will use grayscales only. A +# value of 255 will produce the most vivid colors. +# Minimum value: 0, maximum value: 255, default value: 100. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_COLORSTYLE_SAT = 100 + +# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to the +# luminance component of the colors in the HTML output. Values below 100 +# gradually make the output lighter, whereas values above 100 make the output +# darker. The value divided by 100 is the actual gamma applied, so 80 represents +# a gamma of 0.8, The value 220 represents a gamma of 2.2, and 100 does not +# change the gamma. +# Minimum value: 40, maximum value: 240, default value: 80. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_COLORSTYLE_GAMMA = 80 + +# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML +# page will contain the date and time when the page was generated. Setting this +# to YES can help to show when doxygen was last run and thus if the +# documentation is up to date. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_TIMESTAMP = NO + +# If the HTML_DYNAMIC_MENUS tag is set to YES then the generated HTML +# documentation will contain a main index with vertical navigation menus that +# are dynamically created via JavaScript. If disabled, the navigation index will +# consists of multiple levels of tabs that are statically embedded in every HTML +# page. Disable this option to support browsers that do not have JavaScript, +# like the Qt help browser. +# The default value is: YES. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_DYNAMIC_MENUS = YES + +# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML +# documentation will contain sections that can be hidden and shown after the +# page has loaded. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_DYNAMIC_SECTIONS = NO + +# With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries +# shown in the various tree structured indices initially; the user can expand +# and collapse entries dynamically later on. Doxygen will expand the tree to +# such a level that at most the specified number of entries are visible (unless +# a fully collapsed tree already exceeds this amount). So setting the number of +# entries 1 will produce a full collapsed tree by default. 0 is a special value +# representing an infinite number of entries and will result in a full expanded +# tree by default. +# Minimum value: 0, maximum value: 9999, default value: 100. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_INDEX_NUM_ENTRIES = 100 + +# If the GENERATE_DOCSET tag is set to YES, additional index files will be +# generated that can be used as input for Apple's Xcode 3 integrated development +# environment (see: +# https://developer.apple.com/xcode/), introduced with OSX 10.5 (Leopard). To +# create a documentation set, doxygen will generate a Makefile in the HTML +# output directory. Running make will produce the docset in that directory and +# running make install will install the docset in +# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at +# startup. See https://developer.apple.com/library/archive/featuredarticles/Doxy +# genXcode/_index.html for more information. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_DOCSET = NO + +# This tag determines the name of the docset feed. A documentation feed provides +# an umbrella under which multiple documentation sets from a single provider +# (such as a company or product suite) can be grouped. +# The default value is: Doxygen generated docs. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_FEEDNAME = "Doxygen generated docs" + +# This tag specifies a string that should uniquely identify the documentation +# set bundle. This should be a reverse domain-name style string, e.g. +# com.mycompany.MyDocSet. Doxygen will append .docset to the name. +# The default value is: org.doxygen.Project. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_BUNDLE_ID = org.doxygen.Project + +# The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify +# the documentation publisher. This should be a reverse domain-name style +# string, e.g. com.mycompany.MyDocSet.documentation. +# The default value is: org.doxygen.Publisher. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_PUBLISHER_ID = org.doxygen.Publisher + +# The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher. +# The default value is: Publisher. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_PUBLISHER_NAME = Publisher + +# If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three +# additional HTML index files: index.hhp, index.hhc, and index.hhk. The +# index.hhp is a project file that can be read by Microsoft's HTML Help Workshop +# (see: +# https://www.microsoft.com/en-us/download/details.aspx?id=21138) on Windows. +# +# The HTML Help Workshop contains a compiler that can convert all HTML output +# generated by doxygen into a single compiled HTML file (.chm). Compiled HTML +# files are now used as the Windows 98 help format, and will replace the old +# Windows help format (.hlp) on all Windows platforms in the future. Compressed +# HTML files also contain an index, a table of contents, and you can search for +# words in the documentation. The HTML workshop also contains a viewer for +# compressed HTML files. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_HTMLHELP = NO + +# The CHM_FILE tag can be used to specify the file name of the resulting .chm +# file. You can add a path in front of the file if the result should not be +# written to the html output directory. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +CHM_FILE = + +# The HHC_LOCATION tag can be used to specify the location (absolute path +# including file name) of the HTML help compiler (hhc.exe). If non-empty, +# doxygen will try to run the HTML help compiler on the generated index.hhp. +# The file has to be specified with full path. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +HHC_LOCATION = + +# The GENERATE_CHI flag controls if a separate .chi index file is generated +# (YES) or that it should be included in the main .chm file (NO). +# The default value is: NO. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +GENERATE_CHI = NO + +# The CHM_INDEX_ENCODING is used to encode HtmlHelp index (hhk), content (hhc) +# and project file content. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +CHM_INDEX_ENCODING = + +# The BINARY_TOC flag controls whether a binary table of contents is generated +# (YES) or a normal table of contents (NO) in the .chm file. Furthermore it +# enables the Previous and Next buttons. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +BINARY_TOC = NO + +# The TOC_EXPAND flag can be set to YES to add extra items for group members to +# the table of contents of the HTML help documentation and to the tree view. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +TOC_EXPAND = NO + +# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and +# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that +# can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help +# (.qch) of the generated HTML documentation. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_QHP = NO + +# If the QHG_LOCATION tag is specified, the QCH_FILE tag can be used to specify +# the file name of the resulting .qch file. The path specified is relative to +# the HTML output folder. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QCH_FILE = + +# The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help +# Project output. For more information please see Qt Help Project / Namespace +# (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#namespace). +# The default value is: org.doxygen.Project. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_NAMESPACE = org.doxygen.Project + +# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt +# Help Project output. For more information please see Qt Help Project / Virtual +# Folders (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#virtual-folders). +# The default value is: doc. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_VIRTUAL_FOLDER = doc + +# If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom +# filter to add. For more information please see Qt Help Project / Custom +# Filters (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-filters). +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_CUST_FILTER_NAME = + +# The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the +# custom filter to add. For more information please see Qt Help Project / Custom +# Filters (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-filters). +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_CUST_FILTER_ATTRS = + +# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this +# project's filter section matches. Qt Help Project / Filter Attributes (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#filter-attributes). +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_SECT_FILTER_ATTRS = + +# The QHG_LOCATION tag can be used to specify the location (absolute path +# including file name) of Qt's qhelpgenerator. If non-empty doxygen will try to +# run qhelpgenerator on the generated .qhp file. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHG_LOCATION = + +# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files will be +# generated, together with the HTML files, they form an Eclipse help plugin. To +# install this plugin and make it available under the help contents menu in +# Eclipse, the contents of the directory containing the HTML and XML files needs +# to be copied into the plugins directory of eclipse. The name of the directory +# within the plugins directory should be the same as the ECLIPSE_DOC_ID value. +# After copying Eclipse needs to be restarted before the help appears. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_ECLIPSEHELP = NO + +# A unique identifier for the Eclipse help plugin. When installing the plugin +# the directory name containing the HTML and XML files should also have this +# name. Each documentation set should have its own identifier. +# The default value is: org.doxygen.Project. +# This tag requires that the tag GENERATE_ECLIPSEHELP is set to YES. + +ECLIPSE_DOC_ID = org.doxygen.Project + +# If you want full control over the layout of the generated HTML pages it might +# be necessary to disable the index and replace it with your own. The +# DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) at top +# of each HTML page. A value of NO enables the index and the value YES disables +# it. Since the tabs in the index contain the same information as the navigation +# tree, you can set this option to YES if you also set GENERATE_TREEVIEW to YES. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +DISABLE_INDEX = NO + +# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index +# structure should be generated to display hierarchical information. If the tag +# value is set to YES, a side panel will be generated containing a tree-like +# index structure (just like the one that is generated for HTML Help). For this +# to work a browser that supports JavaScript, DHTML, CSS and frames is required +# (i.e. any modern browser). Windows users are probably better off using the +# HTML help feature. Via custom style sheets (see HTML_EXTRA_STYLESHEET) one can +# further fine-tune the look of the index. As an example, the default style +# sheet generated by doxygen has an example that shows how to put an image at +# the root of the tree instead of the PROJECT_NAME. Since the tree basically has +# the same information as the tab index, you could consider setting +# DISABLE_INDEX to YES when enabling this option. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_TREEVIEW = YES + +# The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that +# doxygen will group on one line in the generated HTML documentation. +# +# Note that a value of 0 will completely suppress the enum values from appearing +# in the overview section. +# Minimum value: 0, maximum value: 20, default value: 4. +# This tag requires that the tag GENERATE_HTML is set to YES. + +ENUM_VALUES_PER_LINE = 4 + +# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be used +# to set the initial width (in pixels) of the frame in which the tree is shown. +# Minimum value: 0, maximum value: 1500, default value: 250. +# This tag requires that the tag GENERATE_HTML is set to YES. + +TREEVIEW_WIDTH = 250 + +# If the EXT_LINKS_IN_WINDOW option is set to YES, doxygen will open links to +# external symbols imported via tag files in a separate window. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +EXT_LINKS_IN_WINDOW = NO + +# If the HTML_FORMULA_FORMAT option is set to svg, doxygen will use the pdf2svg +# tool (see https://github.com/dawbarton/pdf2svg) or inkscape (see +# https://inkscape.org) to generate formulas as SVG images instead of PNGs for +# the HTML output. These images will generally look nicer at scaled resolutions. +# Possible values are: png (the default) and svg (looks nicer but requires the +# pdf2svg or inkscape tool). +# The default value is: png. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_FORMULA_FORMAT = png + +# Use this tag to change the font size of LaTeX formulas included as images in +# the HTML documentation. When you change the font size after a successful +# doxygen run you need to manually remove any form_*.png images from the HTML +# output directory to force them to be regenerated. +# Minimum value: 8, maximum value: 50, default value: 10. +# This tag requires that the tag GENERATE_HTML is set to YES. + +FORMULA_FONTSIZE = 10 + +# Use the FORMULA_TRANSPARENT tag to determine whether or not the images +# generated for formulas are transparent PNGs. Transparent PNGs are not +# supported properly for IE 6.0, but are supported on all modern browsers. +# +# Note that when changing this option you need to delete any form_*.png files in +# the HTML output directory before the changes have effect. +# The default value is: YES. +# This tag requires that the tag GENERATE_HTML is set to YES. + +FORMULA_TRANSPARENT = YES + +# The FORMULA_MACROFILE can contain LaTeX \newcommand and \renewcommand commands +# to create new LaTeX commands to be used in formulas as building blocks. See +# the section "Including formulas" for details. + +FORMULA_MACROFILE = + +# Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see +# https://www.mathjax.org) which uses client side JavaScript for the rendering +# instead of using pre-rendered bitmaps. Use this if you do not have LaTeX +# installed or if you want to formulas look prettier in the HTML output. When +# enabled you may also need to install MathJax separately and configure the path +# to it using the MATHJAX_RELPATH option. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +USE_MATHJAX = NO + +# When MathJax is enabled you can set the default output format to be used for +# the MathJax output. See the MathJax site (see: +# http://docs.mathjax.org/en/v2.7-latest/output.html) for more details. +# Possible values are: HTML-CSS (which is slower, but has the best +# compatibility), NativeMML (i.e. MathML) and SVG. +# The default value is: HTML-CSS. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_FORMAT = HTML-CSS + +# When MathJax is enabled you need to specify the location relative to the HTML +# output directory using the MATHJAX_RELPATH option. The destination directory +# should contain the MathJax.js script. For instance, if the mathjax directory +# is located at the same level as the HTML output directory, then +# MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax +# Content Delivery Network so you can quickly see the result without installing +# MathJax. However, it is strongly recommended to install a local copy of +# MathJax from https://www.mathjax.org before deployment. +# The default value is: https://cdn.jsdelivr.net/npm/mathjax@2. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_RELPATH = https://cdn.jsdelivr.net/npm/mathjax@2 + +# The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax +# extension names that should be enabled during MathJax rendering. For example +# MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_EXTENSIONS = + +# The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces +# of code that will be used on startup of the MathJax code. See the MathJax site +# (see: +# http://docs.mathjax.org/en/v2.7-latest/output.html) for more details. For an +# example see the documentation. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_CODEFILE = + +# When the SEARCHENGINE tag is enabled doxygen will generate a search box for +# the HTML output. The underlying search engine uses javascript and DHTML and +# should work on any modern browser. Note that when using HTML help +# (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets (GENERATE_DOCSET) +# there is already a search function so this one should typically be disabled. +# For large projects the javascript based search engine can be slow, then +# enabling SERVER_BASED_SEARCH may provide a better solution. It is possible to +# search using the keyboard; to jump to the search box use + S +# (what the is depends on the OS and browser, but it is typically +# , /