From e14a1c16e74bcf982f19230489be10ace7e3c3ad Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 21 Apr 2021 14:42:01 +0800 Subject: [PATCH 1/4] change op_mapping and dump_task namespace. --- ge/CMakeLists.txt | 3 +- ge/common/dump/dump_op.cc | 20 +++---- ge/common/dump/dump_op.h | 10 ++-- ge/common/dump/exception_dumper.cc | 16 ++--- ...op_mapping_info.proto => op_mapping.proto} | 2 +- ge/executor/CMakeLists.txt | 40 +------------ ge/executor/proto/dump_task.proto | 2 +- ...op_mapping_info.proto => op_mapping.proto} | 2 +- ge/graph/load/model_manager/data_dumper.cc | 60 ++++++++++--------- ge/graph/load/model_manager/data_dumper.h | 33 +++++----- ge/proto/dump_task.proto | 2 +- ...op_mapping_info.proto => op_mapping.proto} | 2 +- tests/ut/ge/CMakeLists.txt | 2 +- .../ut/ge/graph/load/data_dumper_unittest.cc | 25 +++----- 14 files changed, 87 insertions(+), 132 deletions(-) rename ge/common/proto/{op_mapping_info.proto => op_mapping.proto} (97%) rename ge/executor/proto/{op_mapping_info.proto => op_mapping.proto} (97%) rename ge/proto/{op_mapping_info.proto => op_mapping.proto} (97%) diff --git a/ge/CMakeLists.txt b/ge/CMakeLists.txt index 1285d6e6..41de6199 100755 --- a/ge/CMakeLists.txt +++ b/ge/CMakeLists.txt @@ -11,8 +11,6 @@ elseif (ENABLE_D) endif () set(PROTO_LIST - "${METADEF_DIR}/proto/dump_task.proto" - "${METADEF_DIR}/proto/op_mapping_info.proto" "${METADEF_DIR}/proto/fusion_model.proto" "${GE_CODE_DIR}/ge/proto/optimizer_priority.proto" ) @@ -27,6 +25,7 @@ set(PROTO_HEADER_LIST "${METADEF_DIR}/proto/insert_op.proto" "${METADEF_DIR}/proto/ge_ir.proto" "${METADEF_DIR}/proto/fwk_adapter.proto" + "${METADEF_DIR}/proto/op_mapping.proto" ) protobuf_generate(ge PROTO_SRCS PROTO_HDRS ${PROTO_LIST}) diff --git a/ge/common/dump/dump_op.cc b/ge/common/dump/dump_op.cc index 1ce37b02..8af8781e 100755 --- a/ge/common/dump/dump_op.cc +++ b/ge/common/dump/dump_op.cc @@ -26,7 +26,7 @@ #include "graph/op_desc.h" #include "graph/utils/tensor_utils.h" #include "proto/ge_ir.pb.h" -#include "proto/op_mapping_info.pb.h" +#include "proto/op_mapping.pb.h" #include "runtime/mem.h" #include "aicpu/common/aicpu_task_struct.h" @@ -64,7 +64,7 @@ void DumpOp::SetDynamicModelInfo(const string &dynamic_model_name, const string } static void SetOpMappingLoopAddr(uintptr_t step_id, uintptr_t loop_per_iter, uintptr_t loop_cond, - aicpu::dump::OpMappingInfo &op_mapping_info) { + toolkit::aicpu::dump::OpMappingInfo &op_mapping_info) { if (step_id != 0) { GELOGI("step_id exists."); op_mapping_info.set_step_id_addr(static_cast(step_id)); @@ -87,11 +87,11 @@ static void SetOpMappingLoopAddr(uintptr_t step_id, uintptr_t loop_per_iter, uin } } -Status DumpOp::DumpOutput(aicpu::dump::Task &task) { +Status DumpOp::DumpOutput(toolkit::aicpu::dump::Task &task) { GELOGI("Start dump output in Launch dump op"); const auto &output_descs = op_desc_->GetAllOutputsDesc(); for (size_t i = 0; i < output_descs.size(); ++i) { - aicpu::dump::Output output; + toolkit::aicpu::dump::Output output; output.set_data_type(static_cast(DataTypeUtil::GetIrDataType(output_descs.at(i).GetDataType()))); output.set_format(static_cast(output_descs.at(i).GetFormat())); for (auto dim : output_descs.at(i).GetShape().GetDims()) { @@ -113,11 +113,11 @@ Status DumpOp::DumpOutput(aicpu::dump::Task &task) { return SUCCESS; } -Status DumpOp::DumpInput(aicpu::dump::Task &task) { +Status DumpOp::DumpInput(toolkit::aicpu::dump::Task &task) { GELOGI("Start dump input in Launch dump op"); const auto &input_descs = op_desc_->GetAllInputsDesc(); for (size_t i = 0; i < input_descs.size(); ++i) { - aicpu::dump::Input input; + toolkit::aicpu::dump::Input input; input.set_data_type(static_cast(DataTypeUtil::GetIrDataType(input_descs.at(i).GetDataType()))); input.set_format(static_cast(input_descs.at(i).GetFormat())); @@ -149,7 +149,7 @@ void DumpOp::SetDumpInfo(const DumpProperties &dump_properties, const OpDescPtr stream_ = stream; } -Status DumpOp::ExecutorDumpOp(aicpu::dump::OpMappingInfo &op_mapping_info) { +Status DumpOp::ExecutorDumpOp(toolkit::aicpu::dump::OpMappingInfo &op_mapping_info) { std::string proto_msg; size_t proto_size = op_mapping_info.ByteSizeLong(); bool ret = op_mapping_info.SerializeToString(&proto_msg); @@ -203,7 +203,7 @@ Status DumpOp::ExecutorDumpOp(aicpu::dump::OpMappingInfo &op_mapping_info) { return SUCCESS; } -Status DumpOp::SetDumpModelName(aicpu::dump::OpMappingInfo &op_mapping_info) { +Status DumpOp::SetDumpModelName(toolkit::aicpu::dump::OpMappingInfo &op_mapping_info) { if (dynamic_model_name_.empty() && dynamic_om_name_.empty()) { GELOGI("Single op dump, no need set model name"); return SUCCESS; @@ -242,7 +242,7 @@ Status DumpOp::LaunchDumpOp() { device_id); return ACL_ERROR_GE_INTERNAL_ERROR; } - aicpu::dump::OpMappingInfo op_mapping_info; + toolkit::aicpu::dump::OpMappingInfo op_mapping_info; auto dump_path = dump_properties_.GetDumpPath() + std::to_string(device_id) + "/"; op_mapping_info.set_dump_path(dump_path); op_mapping_info.set_flag(kAicpuLoadFlag); @@ -261,7 +261,7 @@ Status DumpOp::LaunchDumpOp() { if (rt_ret != RT_ERROR_NONE) { GELOGW("call rtGetTaskIdAndStreamID failed, ret = 0x%X", rt_ret); } - aicpu::dump::Task task; + toolkit::aicpu::dump::Task task; task.set_task_id(task_id); task.set_stream_id(stream_id); task.mutable_op()->set_op_name(op_desc_->GetName()); diff --git a/ge/common/dump/dump_op.h b/ge/common/dump/dump_op.h index 4d322bee..b664495a 100755 --- a/ge/common/dump/dump_op.h +++ b/ge/common/dump/dump_op.h @@ -21,7 +21,7 @@ #include "common/ge_inner_error_codes.h" #include "common/properties_manager.h" -#include "proto/op_mapping_info.pb.h" +#include "proto/op_mapping.pb.h" #include "runtime/stream.h" namespace ge { @@ -37,10 +37,10 @@ class DumpOp { void SetDynamicModelInfo(const string &dynamic_model_name, const string &dynamic_om_name, uint32_t dynamic_model_id); private: - Status ExecutorDumpOp(aicpu::dump::OpMappingInfo &op_mapping_info); - Status DumpOutput(aicpu::dump::Task &task); - Status DumpInput(aicpu::dump::Task &task); - Status SetDumpModelName(aicpu::dump::OpMappingInfo &op_mapping_info); + Status ExecutorDumpOp(toolkit::aicpu::dump::OpMappingInfo &op_mapping_info); + Status DumpOutput(toolkit::aicpu::dump::Task &task); + Status DumpInput(toolkit::aicpu::dump::Task &task); + Status SetDumpModelName(toolkit::aicpu::dump::OpMappingInfo &op_mapping_info); DumpProperties dump_properties_; OpDescPtr op_desc_; diff --git a/ge/common/dump/exception_dumper.cc b/ge/common/dump/exception_dumper.cc index bed389a7..c8ec3d35 100644 --- a/ge/common/dump/exception_dumper.cc +++ b/ge/common/dump/exception_dumper.cc @@ -43,15 +43,15 @@ static void ReplaceStringElem(std::string &str) { }); } -static void SetDumpData(const ge::OpDescInfo &op_desc_info, toolkit::dumpdata::DumpData &dump_data) { +static void SetDumpData(const ge::OpDescInfo &op_desc_info, toolkit::dump::DumpData &dump_data) { dump_data.set_version("2.0"); dump_data.set_dump_time(GetNowTime()); dump_data.set_op_name(op_desc_info.op_name); for (size_t i = 0; i < op_desc_info.input_format.size(); ++i) { - toolkit::dumpdata::OpInput input; - input.set_data_type(toolkit::dumpdata::OutputDataType( + toolkit::dump::OpInput input; + input.set_data_type(toolkit::dump::OutputDataType( ge::DataTypeUtil::GetIrDataType(op_desc_info.input_data_type[i]))); - input.set_format(toolkit::dumpdata::OutputFormat(op_desc_info.input_format[i])); + input.set_format(toolkit::dump::OutputFormat(op_desc_info.input_format[i])); for (auto dim : op_desc_info.input_shape[i]) { input.mutable_shape()->add_dim(dim); } @@ -61,10 +61,10 @@ static void SetDumpData(const ge::OpDescInfo &op_desc_info, toolkit::dumpdata::D } for (size_t j = 0; j < op_desc_info.output_format.size(); ++j) { - toolkit::dumpdata::OpOutput output; - output.set_data_type(toolkit::dumpdata::OutputDataType( + toolkit::dump::OpOutput output; + output.set_data_type(toolkit::dump::OutputDataType( ge::DataTypeUtil::GetIrDataType(op_desc_info.output_data_type[j]))); - output.set_format(toolkit::dumpdata::OutputFormat(op_desc_info.output_format[j])); + output.set_format(toolkit::dump::OutputFormat(op_desc_info.output_format[j])); for (auto dim : op_desc_info.output_shape[j]) { output.mutable_shape()->add_dim(dim); } @@ -148,7 +148,7 @@ Status ExceptionDumper::DumpExceptionInfo(const std::vector &ex for (const rtExceptionInfo &iter : exception_infos) { OpDescInfo op_desc_info; if (GetOpDescInfo(iter.streamid, iter.taskid, op_desc_info)) { - toolkit::dumpdata::DumpData dump_data; + toolkit::dump::DumpData dump_data; SetDumpData(op_desc_info, dump_data); uint64_t now_time = GetNowTime(); std::string op_name = op_desc_info.op_name; diff --git a/ge/common/proto/op_mapping_info.proto b/ge/common/proto/op_mapping.proto similarity index 97% rename from ge/common/proto/op_mapping_info.proto rename to ge/common/proto/op_mapping.proto index 7fb6f84b..d626eb49 100644 --- a/ge/common/proto/op_mapping_info.proto +++ b/ge/common/proto/op_mapping.proto @@ -1,5 +1,5 @@ syntax = "proto3"; -package aicpu.dump; +package toolkit.aicpu.dump; message Shape { repeated uint64 dim = 1; diff --git a/ge/executor/CMakeLists.txt b/ge/executor/CMakeLists.txt index a2181578..1782d497 100644 --- a/ge/executor/CMakeLists.txt +++ b/ge/executor/CMakeLists.txt @@ -3,46 +3,12 @@ set(PROTO_LIST "${METADEF_DIR}/proto/ge_ir.proto" "${METADEF_DIR}/proto/insert_op.proto" "${METADEF_DIR}/proto/task.proto" -) - -set(DUMP_PROTO_LIST + "${METADEF_DIR}/proto/op_mapping.proto" "${METADEF_DIR}/proto/dump_task.proto" - "${METADEF_DIR}/proto/op_mapping_info.proto" ) protobuf_generate(ge PROTO_SRCS PROTO_HDRS ${PROTO_LIST}) protobuf_generate(ge_static PROTO_STATIC_SRCS PROTO_STATIC_HDRS ${PROTO_LIST}) -protobuf_generate(ge_dump DUMP_PROTO_SRCS DUMP_PROTO_HDRS ${DUMP_PROTO_LIST}) - -############ libge_proto_dump.a ############ -add_library(ge_proto_dump STATIC - ${DUMP_PROTO_SRCS} -) - -target_compile_definitions(ge_proto_dump PRIVATE - PROTOBUF_INLINE_NOT_IN_HEADERS=0 - google=ascend_private - $,OS_TYPE=WIN,OS_TYPE=0> - $<$:SECUREC_USING_STD_SECURE_LIB=0 NOMINMAX> -) - -target_compile_options(ge_proto_dump PRIVATE - -O2 - -fno-common - -fvisibility=hidden - $<$,$>:/MTd> - $<$,$>:/MT> -) - -target_link_libraries(ge_proto_dump PRIVATE - $ - ascend_protobuf_static -) - -target_link_options(ge_proto_dump PRIVATE - -Wl,-Bsymbolic -) -################################################################## set(SRC_LIST "ge_executor.cc" @@ -227,7 +193,6 @@ target_include_directories(ge_executor SYSTEM PRIVATE ${METADEF_DIR}/inc/external/graph ${METADEF_DIR}/inc/graph ${CMAKE_BINARY_DIR} - ${CMAKE_BINARY_DIR}/proto/ge_dump ${CMAKE_BINARY_DIR}/proto/ge_static #### yellow zone #### ${GE_CODE_DIR}/../inc @@ -240,7 +205,6 @@ target_link_libraries(ge_executor PRIVATE $ json ascend_protobuf_static - ge_proto_dump c_sec $<$>:-lrt> -ldl @@ -275,7 +239,6 @@ target_include_directories(ge_executor_shared PRIVATE ${METADEF_DIR}/inc/external/graph ${METADEF_DIR}/inc/graph ${CMAKE_BINARY_DIR} - ${CMAKE_BINARY_DIR}/proto/ge_dump ${CMAKE_BINARY_DIR}/proto/ge #### yellow zone #### ${GE_CODE_DIR}/../inc @@ -291,7 +254,6 @@ target_link_options(ge_executor_shared PRIVATE target_link_libraries(ge_executor_shared PRIVATE $ - ge_proto_dump -Wl,--no-as-needed ge_common runtime diff --git a/ge/executor/proto/dump_task.proto b/ge/executor/proto/dump_task.proto index ee1c6f47..a2411ddb 100644 --- a/ge/executor/proto/dump_task.proto +++ b/ge/executor/proto/dump_task.proto @@ -1,5 +1,5 @@ syntax = "proto3"; -package toolkit.dumpdata; +package toolkit.dump; enum OutputDataType { DT_UNDEFINED = 0; diff --git a/ge/executor/proto/op_mapping_info.proto b/ge/executor/proto/op_mapping.proto similarity index 97% rename from ge/executor/proto/op_mapping_info.proto rename to ge/executor/proto/op_mapping.proto index 7fb6f84b..d626eb49 100644 --- a/ge/executor/proto/op_mapping_info.proto +++ b/ge/executor/proto/op_mapping.proto @@ -1,5 +1,5 @@ syntax = "proto3"; -package aicpu.dump; +package toolkit.aicpu.dump; message Shape { repeated uint64 dim = 1; diff --git a/ge/graph/load/model_manager/data_dumper.cc b/ge/graph/load/model_manager/data_dumper.cc index 0f23fa2e..ac256526 100644 --- a/ge/graph/load/model_manager/data_dumper.cc +++ b/ge/graph/load/model_manager/data_dumper.cc @@ -35,7 +35,7 @@ #include "graph/utils/tensor_utils.h" #include "proto/dump_task.pb.h" #include "proto/ge_ir.pb.h" -#include "proto/op_mapping_info.pb.h" +#include "proto/op_mapping.pb.h" #include "runtime/base.h" #include "runtime/mem.h" @@ -217,7 +217,7 @@ void DataDumper::SaveDumpTask(uint32_t task_id, uint32_t stream_id, const std::s } static void SetOpMappingLoopAddr(uintptr_t step_id, uintptr_t loop_per_iter, uintptr_t loop_cond, - aicpu::dump::OpMappingInfo &op_mapping_info) { + toolkit::aicpu::dump::OpMappingInfo &op_mapping_info) { if (step_id != 0) { GELOGI("step_id exists."); op_mapping_info.set_step_id_addr(static_cast(step_id)); @@ -234,7 +234,8 @@ static void SetOpMappingLoopAddr(uintptr_t step_id, uintptr_t loop_per_iter, uin } } -Status DataDumper::GenerateOutput(aicpu::dump::Output &output, const OpDesc::Vistor &tensor_descs, +Status DataDumper::GenerateOutput(toolkit::aicpu::dump::Output &output, + const OpDesc::Vistor &tensor_descs, const uintptr_t &addr, size_t index) { output.set_data_type(static_cast(GetIrDataType(tensor_descs.at(index).GetDataType()))); output.set_format(static_cast(tensor_descs.at(index).GetFormat())); @@ -265,7 +266,8 @@ Status DataDumper::GenerateOutput(aicpu::dump::Output &output, const OpDesc::Vis return SUCCESS; } -Status DataDumper::DumpRefOutput(const DataDumper::InnerDumpInfo &inner_dump_info, aicpu::dump::Output &output, +Status DataDumper::DumpRefOutput(const DataDumper::InnerDumpInfo &inner_dump_info, + toolkit::aicpu::dump::Output &output, size_t i, const std::string &node_name_index) { std::string dump_op_name; std::string input_or_output; @@ -306,7 +308,7 @@ Status DataDumper::DumpRefOutput(const DataDumper::InnerDumpInfo &inner_dump_inf return SUCCESS; } -Status DataDumper::DumpOutputWithTask(const InnerDumpInfo &inner_dump_info, aicpu::dump::Task &task) { +Status DataDumper::DumpOutputWithTask(const InnerDumpInfo &inner_dump_info, toolkit::aicpu::dump::Task &task) { const auto &output_descs = inner_dump_info.op->GetAllOutputsDesc(); const std::vector output_addrs = ModelUtils::GetOutputDataAddrs(*runtime_param_, inner_dump_info.op); std::vector v_memory_type; @@ -318,7 +320,7 @@ Status DataDumper::DumpOutputWithTask(const InnerDumpInfo &inner_dump_info, aicp size_t no_need_dump_output_num = 0; for (size_t i = 0; i < output_descs.size(); ++i) { - aicpu::dump::Output output; + toolkit::aicpu::dump::Output output; std::string node_name_index; const auto &output_desc = output_descs.at(i); int32_t calc_type = 0; @@ -370,14 +372,14 @@ Status DataDumper::DumpOutputWithTask(const InnerDumpInfo &inner_dump_info, aicp return SUCCESS; } -Status DataDumper::DumpOutput(const InnerDumpInfo &inner_dump_info, aicpu::dump::Task &task) { +Status DataDumper::DumpOutput(const InnerDumpInfo &inner_dump_info, toolkit::aicpu::dump::Task &task) { GELOGI("Start dump output"); if (inner_dump_info.is_task) { // tbe or aicpu op, these ops are with task return DumpOutputWithTask(inner_dump_info, task); } // else data, const or variable op - aicpu::dump::Output output; + toolkit::aicpu::dump::Output output; auto output_tensor = inner_dump_info.op->GetOutputDescPtr(inner_dump_info.output_anchor_index); const std::vector output_addrs = ModelUtils::GetOutputDataAddrs(*runtime_param_, inner_dump_info.op); if (output_tensor == nullptr) { @@ -422,7 +424,7 @@ Status DataDumper::DumpOutput(const InnerDumpInfo &inner_dump_info, aicpu::dump: return SUCCESS; } -Status DataDumper::GenerateInput(aicpu::dump::Input &input, const OpDesc::Vistor &tensor_descs, +Status DataDumper::GenerateInput(toolkit::aicpu::dump::Input &input, const OpDesc::Vistor &tensor_descs, const uintptr_t &addr, size_t index) { input.set_data_type(static_cast(GetIrDataType(tensor_descs.at(index).GetDataType()))); input.set_format(static_cast(tensor_descs.at(index).GetFormat())); @@ -447,8 +449,8 @@ Status DataDumper::GenerateInput(aicpu::dump::Input &input, const OpDesc::Vistor return SUCCESS; } -Status DataDumper::DumpRefInput(const DataDumper::InnerDumpInfo &inner_dump_info, aicpu::dump::Input &input, size_t i, - const std::string &node_name_index) { +Status DataDumper::DumpRefInput(const DataDumper::InnerDumpInfo &inner_dump_info, toolkit::aicpu::dump::Input &input, + size_t i, const std::string &node_name_index) { std::string dump_op_name; std::string input_or_output; size_t index; @@ -488,7 +490,7 @@ Status DataDumper::DumpRefInput(const DataDumper::InnerDumpInfo &inner_dump_info return SUCCESS; } -Status DataDumper::DumpInput(const InnerDumpInfo &inner_dump_info, aicpu::dump::Task &task) { +Status DataDumper::DumpInput(const InnerDumpInfo &inner_dump_info, toolkit::aicpu::dump::Task &task) { GELOGI("Start dump input"); const auto &input_descs = inner_dump_info.op->GetAllInputsDesc(); const std::vector input_addrs = ModelUtils::GetInputDataAddrs(*runtime_param_, inner_dump_info.op); @@ -507,7 +509,7 @@ Status DataDumper::DumpInput(const InnerDumpInfo &inner_dump_info, aicpu::dump:: inner_dump_info.op->GetName().c_str(), input_descs.size(), v_memory_type.size()); for (size_t i = 0; i < input_descs.size(); ++i) { - aicpu::dump::Input input; + toolkit::aicpu::dump::Input input; std::string node_name_index; // check dump input tensor desc is redirected by attr ATTR_DATA_DUMP_REF if (AttrUtils::GetStr(&input_descs.at(i), ATTR_DATA_DUMP_REF, node_name_index)) { @@ -538,15 +540,15 @@ Status DataDumper::DumpInput(const InnerDumpInfo &inner_dump_info, aicpu::dump:: return SUCCESS; } -void DataDumper::GenerateOpBuffer(const int64_t &size, aicpu::dump::Task &task) { - aicpu::dump::OpBuffer op_buffer; - op_buffer.set_buffer_type(aicpu::dump::BufferType::L1); +void DataDumper::GenerateOpBuffer(const int64_t &size, toolkit::aicpu::dump::Task &task) { + toolkit::aicpu::dump::OpBuffer op_buffer; + op_buffer.set_buffer_type(toolkit::aicpu::dump::BufferType::L1); op_buffer.set_address(reinterpret_cast(l1_fusion_addr_)); op_buffer.set_size(size); task.mutable_buffer()->Add(std::move(op_buffer)); } -Status DataDumper::ExecuteLoadDumpInfo(aicpu::dump::OpMappingInfo &op_mapping_info) { +Status DataDumper::ExecuteLoadDumpInfo(toolkit::aicpu::dump::OpMappingInfo &op_mapping_info) { std::string proto_str; size_t proto_size = op_mapping_info.ByteSizeLong(); bool ret = op_mapping_info.SerializeToString(&proto_str); @@ -590,7 +592,7 @@ Status DataDumper::ExecuteLoadDumpInfo(aicpu::dump::OpMappingInfo &op_mapping_in return SUCCESS; } -Status DataDumper::ExecuteUnLoadDumpInfo(aicpu::dump::OpMappingInfo &op_mapping_info) { +Status DataDumper::ExecuteUnLoadDumpInfo(toolkit::aicpu::dump::OpMappingInfo &op_mapping_info) { std::string proto_str; size_t proto_size = op_mapping_info.ByteSizeLong(); bool ret = op_mapping_info.SerializeToString(&proto_str); @@ -641,7 +643,7 @@ Status DataDumper::LoadDumpInfo() { GELOGD("op_list_ is empty"); } - aicpu::dump::OpMappingInfo op_mapping_info; + toolkit::aicpu::dump::OpMappingInfo op_mapping_info; auto dump_path = dump_properties_.GetDumpPath() + std::to_string(device_id_) + "/"; op_mapping_info.set_dump_path(dump_path); @@ -670,11 +672,11 @@ Status DataDumper::LoadDumpInfo() { return SUCCESS; } -Status DataDumper::BuildTaskInfo(aicpu::dump::OpMappingInfo &op_mapping_info) { +Status DataDumper::BuildTaskInfo(toolkit::aicpu::dump::OpMappingInfo &op_mapping_info) { for (const auto &op_iter : op_list_) { auto op_desc = op_iter.op; GELOGD("Op %s in model begin to add task in op_mapping_info", op_desc->GetName().c_str()); - aicpu::dump::Task task; + toolkit::aicpu::dump::Task task; task.set_end_graph(false); task.set_task_id(op_iter.task_id); task.set_stream_id(op_iter.stream_id); @@ -722,10 +724,10 @@ Status DataDumper::BuildTaskInfo(aicpu::dump::OpMappingInfo &op_mapping_info) { } void DataDumper::SetEndGraphIdToAicpu(uint32_t task_id, uint32_t stream_id, - aicpu::dump::OpMappingInfo &op_mapping_info) { + toolkit::aicpu::dump::OpMappingInfo &op_mapping_info) { if (dump_properties_.GetDumpMode() == kDumpOutput || dump_properties_.GetDumpMode() == kDumpInput || dump_properties_.GetDumpMode() == kDumpAll) { - aicpu::dump::Task task; + toolkit::aicpu::dump::Task task; task.set_end_graph(true); task.set_task_id(end_graph_task_id_); task.set_stream_id(end_graph_stream_id_); @@ -734,7 +736,7 @@ void DataDumper::SetEndGraphIdToAicpu(uint32_t task_id, uint32_t stream_id, op_mapping_info.mutable_task()->Add(std::move(task)); is_end_graph_ = true; - if (op_mapping_info.model_name_param_case() == aicpu::dump::OpMappingInfo::kModelName) { + if (op_mapping_info.model_name_param_case() == toolkit::aicpu::dump::OpMappingInfo::kModelName) { GELOGI("Add end_graph_info to aicpu, model_name is %s, task_id is %u, stream_id is %u", op_mapping_info.model_name().c_str(), end_graph_task_id_, end_graph_stream_id_); return; @@ -744,10 +746,10 @@ void DataDumper::SetEndGraphIdToAicpu(uint32_t task_id, uint32_t stream_id, } void DataDumper::SetOpDebugIdToAicpu(uint32_t task_id, uint32_t stream_id, void *op_debug_addr, - aicpu::dump::OpMappingInfo &op_mapping_info) { + toolkit::aicpu::dump::OpMappingInfo &op_mapping_info) { if (is_op_debug_) { GELOGI("add op_debug_info to aicpu, task_id is %u, stream_id is %u", task_id, stream_id); - aicpu::dump::Task task; + toolkit::aicpu::dump::Task task; task.set_end_graph(false); task.set_task_id(task_id); task.set_stream_id(stream_id); @@ -755,7 +757,7 @@ void DataDumper::SetOpDebugIdToAicpu(uint32_t task_id, uint32_t stream_id, void task.mutable_op()->set_op_type(OP_TYPE_OP_DEBUG); // set output - aicpu::dump::Output output; + toolkit::aicpu::dump::Output output; output.set_data_type(DT_UINT8); output.set_format(FORMAT_ND); @@ -781,12 +783,12 @@ Status DataDumper::UnloadDumpInfo() { } GELOGI("UnloadDumpInfo start."); - aicpu::dump::OpMappingInfo op_mapping_info; + toolkit::aicpu::dump::OpMappingInfo op_mapping_info; op_mapping_info.set_model_id(model_id_); op_mapping_info.set_flag(kAicpuUnloadFlag); for (const auto &op_iter : op_list_) { - aicpu::dump::Task task; + toolkit::aicpu::dump::Task task; task.set_task_id(op_iter.task_id); task.set_stream_id(op_iter.stream_id); op_mapping_info.mutable_task()->Add(std::move(task)); diff --git a/ge/graph/load/model_manager/data_dumper.h b/ge/graph/load/model_manager/data_dumper.h index 8af07d86..d1714950 100755 --- a/ge/graph/load/model_manager/data_dumper.h +++ b/ge/graph/load/model_manager/data_dumper.h @@ -27,7 +27,7 @@ #include "graph/node.h" #include "graph/compute_graph.h" #include "proto/ge_ir.pb.h" -#include "proto/op_mapping_info.pb.h" +#include "proto/op_mapping.pb.h" #include "runtime/mem.h" #include "task_info/task_info.h" #include "framework/common/ge_types.h" @@ -128,24 +128,25 @@ class DataDumper { DumpProperties dump_properties_; // Build task info of op mapping info - Status BuildTaskInfo(aicpu::dump::OpMappingInfo &op_mapping_info); - Status DumpOutput(const InnerDumpInfo &inner_dump_info, aicpu::dump::Task &task); - Status DumpRefOutput(const DataDumper::InnerDumpInfo &inner_dump_info, aicpu::dump::Output &output, size_t i, - const std::string &node_name_index); - Status DumpOutputWithTask(const InnerDumpInfo &inner_dump_info, aicpu::dump::Task &task); - Status DumpInput(const InnerDumpInfo &inner_dump_info, aicpu::dump::Task &task); - Status DumpRefInput(const DataDumper::InnerDumpInfo &inner_dump_info, aicpu::dump::Input &input, size_t i, - const std::string &node_name_index); - Status ExecuteLoadDumpInfo(aicpu::dump::OpMappingInfo &op_mapping_info); - void SetEndGraphIdToAicpu(uint32_t task_id, uint32_t stream_id, aicpu::dump::OpMappingInfo &op_mapping_info); + Status BuildTaskInfo(toolkit::aicpu::dump::OpMappingInfo &op_mapping_info); + Status DumpOutput(const InnerDumpInfo &inner_dump_info, toolkit::aicpu::dump::Task &task); + Status DumpRefOutput(const DataDumper::InnerDumpInfo &inner_dump_info, toolkit::aicpu::dump::Output &output, + size_t i, const std::string &node_name_index); + Status DumpOutputWithTask(const InnerDumpInfo &inner_dump_info, toolkit::aicpu::dump::Task &task); + Status DumpInput(const InnerDumpInfo &inner_dump_info, toolkit::aicpu::dump::Task &task); + Status DumpRefInput(const DataDumper::InnerDumpInfo &inner_dump_info, toolkit::aicpu::dump::Input &input, + size_t i, const std::string &node_name_index); + Status ExecuteLoadDumpInfo(toolkit::aicpu::dump::OpMappingInfo &op_mapping_info); + void SetEndGraphIdToAicpu(uint32_t task_id, uint32_t stream_id, + toolkit::aicpu::dump::OpMappingInfo &op_mapping_info); void SetOpDebugIdToAicpu(uint32_t task_id, uint32_t stream_id, void *op_debug_addr, - aicpu::dump::OpMappingInfo &op_mapping_info); - Status ExecuteUnLoadDumpInfo(aicpu::dump::OpMappingInfo &op_mapping_info); - Status GenerateInput(aicpu::dump::Input &input, const OpDesc::Vistor &tensor_descs, + toolkit::aicpu::dump::OpMappingInfo &op_mapping_info); + Status ExecuteUnLoadDumpInfo(toolkit::aicpu::dump::OpMappingInfo &op_mapping_info); + Status GenerateInput(toolkit::aicpu::dump::Input &input, const OpDesc::Vistor &tensor_descs, const uintptr_t &addr, size_t index); - Status GenerateOutput(aicpu::dump::Output &output, const OpDesc::Vistor &tensor_descs, + Status GenerateOutput(toolkit::aicpu::dump::Output &output, const OpDesc::Vistor &tensor_descs, const uintptr_t &addr, size_t index); - void GenerateOpBuffer(const int64_t &size, aicpu::dump::Task &task); + void GenerateOpBuffer(const int64_t &size, toolkit::aicpu::dump::Task &task); }; struct DataDumper::InnerDumpInfo { uint32_t task_id; diff --git a/ge/proto/dump_task.proto b/ge/proto/dump_task.proto index ee1c6f47..a2411ddb 100644 --- a/ge/proto/dump_task.proto +++ b/ge/proto/dump_task.proto @@ -1,5 +1,5 @@ syntax = "proto3"; -package toolkit.dumpdata; +package toolkit.dump; enum OutputDataType { DT_UNDEFINED = 0; diff --git a/ge/proto/op_mapping_info.proto b/ge/proto/op_mapping.proto similarity index 97% rename from ge/proto/op_mapping_info.proto rename to ge/proto/op_mapping.proto index 7fb6f84b..d626eb49 100644 --- a/ge/proto/op_mapping_info.proto +++ b/ge/proto/op_mapping.proto @@ -1,5 +1,5 @@ syntax = "proto3"; -package aicpu.dump; +package toolkit.aicpu.dump; message Shape { repeated uint64 dim = 1; diff --git a/tests/ut/ge/CMakeLists.txt b/tests/ut/ge/CMakeLists.txt index 9bf6bcef..5747142e 100755 --- a/tests/ut/ge/CMakeLists.txt +++ b/tests/ut/ge/CMakeLists.txt @@ -24,7 +24,7 @@ set(PROTO_LIST "${GE_CODE_DIR}/metadef/proto/insert_op.proto" "${GE_CODE_DIR}/metadef/proto/dump_task.proto" "${GE_CODE_DIR}/metadef/proto/fwk_adapter.proto" - "${GE_CODE_DIR}/metadef/proto/op_mapping_info.proto" + "${GE_CODE_DIR}/metadef/proto/op_mapping.proto" "${GE_CODE_DIR}/metadef/proto/optimizer_priority.proto" "${GE_CODE_DIR}/metadef/proto/ge_api.proto" "${GE_CODE_DIR}/metadef/proto/tensorflow/attr_value.proto" diff --git a/tests/ut/ge/graph/load/data_dumper_unittest.cc b/tests/ut/ge/graph/load/data_dumper_unittest.cc index 8f46e989..1eb9eb49 100644 --- a/tests/ut/ge/graph/load/data_dumper_unittest.cc +++ b/tests/ut/ge/graph/load/data_dumper_unittest.cc @@ -51,30 +51,19 @@ static ge::OpDescPtr CreateOpDesc(string name = "", string type = "") { return op_desc; } -/* -TEST_F(UtestDataDumper, LoadDumpInfo_no_output_addrs_fail) { +TEST_F(UtestDataDumper, LoadDumpInfo_success) { RuntimeParam rts_param; - DataDumper data_dumper(rts_param); + DataDumper data_dumper(&rts_param); data_dumper.SetModelName("test"); data_dumper.SetModelId(2333); std::shared_ptr op_desc_1(new OpDesc()); op_desc_1->AddOutputDesc("test", GeTensorDesc()); data_dumper.SaveDumpTask(0, 0, op_desc_1, 0); string dump_mode = "output"; + data_dumper.is_op_debug_ = true; data_dumper.dump_properties_.SetDumpMode(dump_mode); - Status ret = data_dumper.LoadDumpInfo(); - EXPECT_EQ(ret, SUCCESS); -} -*/ - -TEST_F(UtestDataDumper, UnloadDumpInfo_success) { - RuntimeParam rts_param; - DataDumper data_dumper(&rts_param); - data_dumper.SetModelName("test"); - data_dumper.SetModelId(2333); - - Status ret = data_dumper.UnloadDumpInfo(); - EXPECT_EQ(ret, SUCCESS); + EXPECT_EQ(data_dumper.LoadDumpInfo(), SUCCESS); + EXPECT_EQ(data_dumper.UnloadDumpInfo(), SUCCESS); } TEST_F(UtestDataDumper, DumpOutputWithTask_success) { @@ -83,7 +72,7 @@ TEST_F(UtestDataDumper, DumpOutputWithTask_success) { data_dumper.SetModelName("test"); data_dumper.SetModelId(2333); - aicpu::dump::Task task; + toolkit::aicpu::dump::Task task; OpDescPtr op_desc = CreateOpDesc("conv", CONVOLUTION); GeTensorDesc tensor_0(GeShape(), FORMAT_NCHW, DT_FLOAT); GeTensorDesc tensor_1(GeShape(), FORMAT_NCHW, DT_FLOAT); @@ -95,5 +84,7 @@ TEST_F(UtestDataDumper, DumpOutputWithTask_success) { inner_dump_info.op = op_desc; Status ret = data_dumper.DumpOutputWithTask(inner_dump_info, task); EXPECT_EQ(ret, SUCCESS); + int64_t task_size = 1; + data_dumper.GenerateOpBuffer(task_size, task); } } // namespace ge From ed317653c925c78c4a6946fea2877773e284ac56 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 22 Apr 2021 10:09:25 +0800 Subject: [PATCH 2/4] Update submodule. --- metadef | 2 +- parser | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/metadef b/metadef index c1aea328..7ad77efb 160000 --- a/metadef +++ b/metadef @@ -1 +1 @@ -Subproject commit c1aea328cc04340188e796e639cd55a907488365 +Subproject commit 7ad77efb16b4a8a8d08345d600b345b7814f8ab6 diff --git a/parser b/parser index 06e784fa..dda9def8 160000 --- a/parser +++ b/parser @@ -1 +1 @@ -Subproject commit 06e784fad01d7e9089cc7e8e0d00fce5b1901886 +Subproject commit dda9def8bdb5177054a3abc132bf376172c3f01f From ddee5049b07afb8d789317dd9d7cc6c50f229c5a Mon Sep 17 00:00:00 2001 From: zhaozhixuan Date: Thu, 22 Apr 2021 11:12:40 +0800 Subject: [PATCH 3/4] Fix ut. --- .../testcase/ge_graph/ge_def_type_unittest.cc | 32 ------------------- .../testcase/ge_graph/ge_tensor_unittest.cc | 12 ------- 2 files changed, 44 deletions(-) diff --git a/tests/ut/common/graph/testcase/ge_graph/ge_def_type_unittest.cc b/tests/ut/common/graph/testcase/ge_graph/ge_def_type_unittest.cc index 10d6dc86..7dc124fe 100644 --- a/tests/ut/common/graph/testcase/ge_graph/ge_def_type_unittest.cc +++ b/tests/ut/common/graph/testcase/ge_graph/ge_def_type_unittest.cc @@ -32,14 +32,6 @@ class UtestGeTestDefType : public testing::Test { void TearDown() {} }; -TEST_F(UtestGeTestDefType, base) { - CompressInfo com1; - com1.set_blockrow(1); - int32_t a = com1.blockrow; - EXPECT_EQ(a, 1); - -} - TEST_F(UtestGeTestDefType, quant) { OpDescPtr desc_ptr1 = std::make_shared("name1", "type1"); EXPECT_EQ(desc_ptr1->AddInputDesc("x", GeTensorDesc(GeShape({1, 16, 16, 16}), FORMAT_NCHW)), GRAPH_SUCCESS); @@ -48,28 +40,4 @@ TEST_F(UtestGeTestDefType, quant) { EXPECT_EQ(OpDescUtils::HasQuantizeFactorParams(desc_ptr1), false); EXPECT_EQ(OpDescUtils::HasQuantizeFactorParams(*desc_ptr1), false); - QuantizeFactorParams q1; - EXPECT_EQ(q1.has_quantize_param(), false); - QuantizeFactor *qf1 = q1.mutable_quantize_param(); - EXPECT_EQ(q1.has_quantize_param(), true); - - string s1 = "value1"; - q1.quantize_param.set_scale_value(s1.data(), s1.size()); - EXPECT_EQ(OpDescUtils::SetQuantizeFactorParams(desc_ptr1, q1), GRAPH_SUCCESS); - QuantizeFactorParams q2; - EXPECT_EQ(OpDescUtils::GetQuantizeFactorParams(desc_ptr1, q2), GRAPH_SUCCESS); - string s2((char *)q2.quantize_param.scale_value.GetData(), q2.quantize_param.scale_value.GetSize()); - EXPECT_EQ(s2, "value1"); - - float f[2] = {1, 2}; - string s(static_cast(static_cast(f)), 2 * sizeof(float)); - q1.quantize_param.set_scale_value(f, 2 * sizeof(float)); - EXPECT_EQ(OpDescUtils::SetQuantizeFactorParams(*desc_ptr1, q1), GRAPH_SUCCESS); - QuantizeFactorParams q3; - EXPECT_EQ(OpDescUtils::GetQuantizeFactorParams(*desc_ptr1, q3), GRAPH_SUCCESS); - Buffer &b = q3.quantize_param.scale_value; - float f1[2]; - memcpy(f1, b.GetData(), b.GetSize()); - EXPECT_EQ(f1[0], 1); - EXPECT_EQ(f1[1], 2); } diff --git a/tests/ut/common/graph/testcase/ge_graph/ge_tensor_unittest.cc b/tests/ut/common/graph/testcase/ge_graph/ge_tensor_unittest.cc index 5c75bd01..aa43ac99 100644 --- a/tests/ut/common/graph/testcase/ge_graph/ge_tensor_unittest.cc +++ b/tests/ut/common/graph/testcase/ge_graph/ge_tensor_unittest.cc @@ -209,18 +209,6 @@ TEST_F(UtestGeTensor, test_tensor_desc_invalid_null) { tensor_desc2.SetDataType(DT_DUAL_SUB_INT8); EXPECT_EQ(tensor_desc2.GetDataType(), DT_DUAL_SUB_INT8); - CompressInfo info; - EXPECT_EQ(TensorUtils::GetCmpsInfo(tensor_desc2, info), GRAPH_FAILED); - TensorUtils::SetCmpsInfo(tensor_desc2, info); - EXPECT_EQ(TensorUtils::GetCmpsInfo(tensor_desc2, info), GRAPH_SUCCESS); - - AllOffsetQuantizeInfo quantize_info; - EXPECT_FALSE(TensorUtils::HasAlloffsetQuantizeInfo(tensor_desc2)); - EXPECT_EQ(TensorUtils::GetAlloffsetQuantizeInfo(tensor_desc2, quantize_info), GRAPH_FAILED); - TensorUtils::SetAlloffsetQuantizeInfo(tensor_desc2, quantize_info); - EXPECT_EQ(TensorUtils::GetAlloffsetQuantizeInfo(tensor_desc2, quantize_info), GRAPH_SUCCESS); - EXPECT_TRUE(TensorUtils::HasAlloffsetQuantizeInfo(tensor_desc2)); - TensorUtils::SetWeightSize(tensor_desc, 100); EXPECT_EQ(TensorUtils::GetWeightSize(tensor_desc), 0); } From 154445dccae1a3061d473adb5bea2185f9003b77 Mon Sep 17 00:00:00 2001 From: zhaozhixuan Date: Thu, 22 Apr 2021 11:48:47 +0800 Subject: [PATCH 4/4] Fix ut. --- tests/depends/error_manager/src/error_manager_stub.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/depends/error_manager/src/error_manager_stub.cc b/tests/depends/error_manager/src/error_manager_stub.cc index 5f62c91b..c2d61b4a 100644 --- a/tests/depends/error_manager/src/error_manager_stub.cc +++ b/tests/depends/error_manager/src/error_manager_stub.cc @@ -18,7 +18,7 @@ using namespace ErrorMessage; -namespace ErrorMessage { +namespace error_message { int FormatErrorMessage(char *str_dst, size_t dst_max, const char *format, ...) { return 1; }