From 20daa5638bc507b320df04e2eda06d86a0eb3209 Mon Sep 17 00:00:00 2001 From: zhou_chao1993 Date: Fri, 26 Feb 2021 16:31:55 +0800 Subject: [PATCH] infer dump --- ge/common/dump/dump_manager.cc | 44 ++++++----- ge/common/dump/dump_manager.h | 10 +-- ge/common/dump/dump_properties.cc | 6 +- ge/common/dump/dump_properties.h | 2 +- ge/common/properties_manager.cc | 20 ----- ge/common/properties_manager.h | 9 --- ge/graph/build/model_builder.cc | 3 +- ge/graph/load/model_manager/davinci_model.h | 2 +- ge/graph/load/model_manager/model_manager.cc | 21 ++--- ge/graph/load/model_manager/model_manager.h | 2 +- ge/graph/manager/graph_manager.cc | 3 +- .../executor/hybrid_model_async_executor.cc | 4 + .../executor/hybrid_model_async_executor.h | 3 + ge/hybrid/executor/hybrid_model_executor.cc | 3 +- .../hybrid_model_pipeline_executor.cc | 3 +- ge/hybrid/hybrid_davinci_model.cc | 11 +++ ge/hybrid/hybrid_davinci_model.h | 2 + ge/hybrid/hybrid_davinci_model_stub.cc | 3 + ge/hybrid/model/hybrid_model.h | 9 +++ .../compiledsubgraph/known_node_executor.cc | 1 + ge/session/inner_session.cc | 7 +- ge/single_op/single_op.cc | 2 + ge/single_op/task/op_task.cc | 30 +++----- tests/ut/ge/CMakeLists.txt | 2 + tests/ut/ge/common/dump_manager_unittest.cc | 76 +++++++++++++++++++ 25 files changed, 188 insertions(+), 90 deletions(-) create mode 100644 tests/ut/ge/common/dump_manager_unittest.cc diff --git a/ge/common/dump/dump_manager.cc b/ge/common/dump/dump_manager.cc index 17019c5a..74324059 100644 --- a/ge/common/dump/dump_manager.cc +++ b/ge/common/dump/dump_manager.cc @@ -22,6 +22,7 @@ namespace { const char *const kDumpOFF = "OFF"; const char *const kDumpoff = "off"; const char *const kDumpOn = "on"; +const uint64_t kInferSessionId = 0; } // namespace namespace ge { FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY DumpManager &DumpManager::GetInstance() { @@ -30,15 +31,14 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY DumpManager &DumpManager::GetIn } FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status DumpManager::SetDumpConf(const DumpConfig &dump_config) { - std::lock_guard lock(mutex_); - dump_properties_.ClearDumpPropertyValue(); - dump_properties_.ClearDumpInfo(); + DumpProperties dump_properties; std::string dump_status; std::string dump_path; std::string dump_mode; std::string dump_op_switch; if (dump_config.dump_status.empty()) { + dump_properties_map_.emplace(kInferSessionId, dump_properties); GELOGI("Dump does not open"); return SUCCESS; } @@ -46,14 +46,16 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status DumpManager::SetDumpConf dump_status = dump_config.dump_status; GELOGI("Dump status is %s", dump_status.c_str()); if (dump_config.dump_status == kDumpoff || dump_config.dump_status == kDumpOFF) { - dump_properties_.ClearDumpPropertyValue(); + dump_properties.ClearDumpPropertyValue(); + dump_properties_map_.emplace(kInferSessionId, dump_properties); return SUCCESS; } - dump_properties_.SetDumpStatus(dump_status); + dump_properties.SetDumpStatus(dump_status); dump_op_switch = dump_config.dump_op_switch; - dump_properties_.SetDumpOpSwitch(dump_op_switch); + dump_properties.SetDumpOpSwitch(dump_op_switch); if (dump_op_switch == kDumpoff && dump_config.dump_list.empty()) { + dump_properties_map_.emplace(kInferSessionId, dump_properties); GELOGE(PARAM_INVALID, "Dump list is invalid,dump_op_switch is %s", dump_op_switch.c_str()); return PARAM_INVALID; } @@ -67,15 +69,15 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status DumpManager::SetDumpConf GELOGI("Dump layer is %s in model", layer.c_str()); dump_layers.insert(layer); } - dump_properties_.AddPropertyValue(model_name, dump_layers); + dump_properties.AddPropertyValue(model_name, dump_layers); } if (dump_op_switch == kDumpOn) { - GELOGI("Start to dump model and single op,dumo op switch is %s", dump_op_switch.c_str()); + GELOGI("Start to dump model and single op,dump op switch is %s", dump_op_switch.c_str()); } else { GELOGI("Only dump model,dump op switch is %s", dump_op_switch.c_str()); } } else { - GELOGI("Only dump single op,dumo op switch is %s", dump_op_switch.c_str()); + GELOGI("Only dump single op,dump op switch is %s", dump_op_switch.c_str()); } dump_path = dump_config.dump_path; @@ -89,27 +91,35 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status DumpManager::SetDumpConf } dump_path = dump_path + CurrentTimeInStr() + "/"; GELOGI("Dump path is %s", dump_path.c_str()); - dump_properties_.SetDumpPath(dump_path); + dump_properties.SetDumpPath(dump_path); dump_mode = dump_config.dump_mode; GELOGI("Dump mode is %s", dump_mode.c_str()); - dump_properties_.SetDumpMode(dump_mode); + dump_properties.SetDumpMode(dump_mode); + dump_properties_map_.emplace(kInferSessionId, dump_properties); return SUCCESS; } -FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY const DumpProperties &DumpManager::GetDumpProperties() { +FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY const DumpProperties &DumpManager::GetDumpProperties( + uint64_t session_id) { std::lock_guard lock(mutex_); - return dump_properties_; + // If session_id is not found in dump_properties_map_, operator[] will insert one. + return dump_properties_map_[session_id]; } -FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void DumpManager::SetModelName(const std::string &model_name) { +FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void DumpManager::AddDumpProperties( + uint64_t session_id, const DumpProperties &dump_properties) { std::lock_guard lock(mutex_); - model_name_ = model_name; + dump_properties_map_.emplace(session_id, dump_properties); } -FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY const std::string &DumpManager::GetModelName() { +FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void DumpManager::RemoveDumpProperties(uint64_t session_id) { std::lock_guard lock(mutex_); - return model_name_; + auto iter = dump_properties_map_.find(session_id); + if (iter != dump_properties_map_.end()) { + dump_properties_map_.erase(iter); + } } + } // namespace ge diff --git a/ge/common/dump/dump_manager.h b/ge/common/dump/dump_manager.h index 53a643f9..095344b7 100644 --- a/ge/common/dump/dump_manager.h +++ b/ge/common/dump/dump_manager.h @@ -28,14 +28,14 @@ class DumpManager { static DumpManager &GetInstance(); Status SetDumpConf(const DumpConfig &dump_config); - const DumpProperties &GetDumpProperties(); - void SetModelName(const std::string &model_name); - const std::string &GetModelName(); + const DumpProperties &GetDumpProperties(uint64_t session_id); + const std::map &GetDumpPropertiesMap() { return dump_properties_map_; } + void AddDumpProperties(uint64_t session_id, const DumpProperties &dump_properties); + void RemoveDumpProperties(uint64_t session_id); private: - DumpProperties dump_properties_; std::mutex mutex_; - std::string model_name_; + std::map dump_properties_map_; }; } // namespace ge #endif // GE_COMMON_DUMP_DUMP_MANAGER_H_ diff --git a/ge/common/dump/dump_properties.cc b/ge/common/dump/dump_properties.cc index a4540367..3fbfd16b 100644 --- a/ge/common/dump/dump_properties.cc +++ b/ge/common/dump/dump_properties.cc @@ -122,6 +122,8 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void DumpProperties::ClearDumpI dump_path_.clear(); dump_step_.clear(); dump_mode_.clear(); + dump_op_switch_.clear(); + dump_status_.clear(); is_op_debug_ = false; op_debug_mode_ = 0; } @@ -201,7 +203,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY const std::string &DumpProperti } FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void DumpProperties::SetDumpOpSwitch( - const std::string dump_op_switch) { + const std::string &dump_op_switch) { dump_op_switch_ = dump_op_switch; } @@ -230,6 +232,8 @@ void DumpProperties::CopyFrom(const DumpProperties &other) { dump_path_ = other.dump_path_; dump_step_ = other.dump_step_; dump_mode_ = other.dump_mode_; + dump_status_ = other.dump_status_; + dump_op_switch_ = other.dump_op_switch_; model_dump_properties_map_ = other.model_dump_properties_map_; is_op_debug_ = other.is_op_debug_; diff --git a/ge/common/dump/dump_properties.h b/ge/common/dump/dump_properties.h index 682d2d08..67f8c00e 100644 --- a/ge/common/dump/dump_properties.h +++ b/ge/common/dump/dump_properties.h @@ -65,7 +65,7 @@ class DumpProperties { const std::string &GetDumpStatus() const; - void SetDumpOpSwitch(const std::string dump_op_switch); + void SetDumpOpSwitch(const std::string &dump_op_switch); const std::string &GetDumpOpSwitch() const; diff --git a/ge/common/properties_manager.cc b/ge/common/properties_manager.cc index 3ca5bd27..eae29e34 100644 --- a/ge/common/properties_manager.cc +++ b/ge/common/properties_manager.cc @@ -165,24 +165,4 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void PropertiesManager::SetProp delimiter = de; } -FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY DumpProperties &PropertiesManager::GetDumpProperties( - uint64_t session_id) { - std::lock_guard lock(mutex_); - // If session_id is not found in dump_properties_map_, operator[] will insert one. - return dump_properties_map_[session_id]; -} - -FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void PropertiesManager::AddDumpProperties( - uint64_t session_id, const DumpProperties &dump_properties) { - std::lock_guard lock(mutex_); - dump_properties_map_.emplace(session_id, dump_properties); -} - -FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void PropertiesManager::RemoveDumpProperties(uint64_t session_id) { - std::lock_guard lock(mutex_); - auto iter = dump_properties_map_.find(session_id); - if (iter != dump_properties_map_.end()) { - dump_properties_map_.erase(iter); - } -} } // namespace ge diff --git a/ge/common/properties_manager.h b/ge/common/properties_manager.h index b4c5aad1..7079eecb 100644 --- a/ge/common/properties_manager.h +++ b/ge/common/properties_manager.h @@ -83,13 +83,6 @@ class PropertiesManager { */ void SetPropertyDelimiter(const std::string &de); - DumpProperties &GetDumpProperties(uint64_t session_id); - - const map &GetDumpPropertiesMap() { return dump_properties_map_; } - - void AddDumpProperties(uint64_t session_id, const DumpProperties &dump_properties); - void RemoveDumpProperties(uint64_t session_id); - private: // Private construct, destructor PropertiesManager(); @@ -111,8 +104,6 @@ class PropertiesManager { std::map properties_map_; std::mutex mutex_; - - std::map dump_properties_map_; }; } // namespace ge diff --git a/ge/graph/build/model_builder.cc b/ge/graph/build/model_builder.cc index ec891f70..78c49057 100755 --- a/ge/graph/build/model_builder.cc +++ b/ge/graph/build/model_builder.cc @@ -19,6 +19,7 @@ #include #include #include "common/ge/ge_util.h" +#include "common/dump/dump_manager.h" #include "framework/common/debug/ge_log.h" #include "graph/anchor.h" #include "graph/attr_value.h" @@ -429,7 +430,7 @@ Status ModelBuilder::BuildModelDef(ge::Model &model) { GE_CHK_BOOL_EXEC(ge::AttrUtils::SetBool(&model, ATTR_NAME_SWITCH_FOR_L1_FUSION, is_l1_fusion_enable_), GELOGE(FAILED, "SetBool of ATTR_NAME_SWITCH_FOR_L1_FUSION failed."); return FAILED); - const DumpProperties &dump_properties = PropertiesManager::Instance().GetDumpProperties(session_id_); + const DumpProperties &dump_properties = DumpManager::GetInstance().GetDumpProperties(session_id_); bool is_op_debug = dump_properties.IsOpDebugOpen(); if (is_op_debug) { if (!ge::AttrUtils::SetBool(&model, ATTR_OP_DEBUG_FLAG, is_op_debug)) { diff --git a/ge/graph/load/model_manager/davinci_model.h b/ge/graph/load/model_manager/davinci_model.h index 5bc3a68e..660d2966 100755 --- a/ge/graph/load/model_manager/davinci_model.h +++ b/ge/graph/load/model_manager/davinci_model.h @@ -536,7 +536,7 @@ class DavinciModel { vector &output_dims) const; // om file name - void SetOmName(string om_name) { om_name_ = om_name; } + void SetOmName(const string &om_name) { om_name_ = om_name; } void SetDumpProperties(const DumpProperties &dump_properties) { data_dumper_.SetDumpProperties(dump_properties); } const DumpProperties &GetDumpProperties() const { return data_dumper_.GetDumpProperties(); } diff --git a/ge/graph/load/model_manager/model_manager.cc b/ge/graph/load/model_manager/model_manager.cc index 512c6e72..b17c65e3 100755 --- a/ge/graph/load/model_manager/model_manager.cc +++ b/ge/graph/load/model_manager/model_manager.cc @@ -55,6 +55,7 @@ const char *const kDeleteCustOp = "deleteCustOp"; const int kTimeSpecNano = 1000000000; const int kTimeSpecMiro = 1000000; const int kOpNameMaxSize = 100; +const uint64_t kInferSessionId = 0; #pragma pack(push, 1) struct CustAicpuSoBuf { uint64_t kernelSoBuf; @@ -278,13 +279,14 @@ ge::Status ModelManager::SetDynamicSize(uint32_t model_id, const std::vector &ge_root_model, +ge::Status ModelManager::DoLoadHybridModelOnline(uint32_t model_id, const string &model_name, const shared_ptr &ge_root_model, const shared_ptr &listener) { auto hybrid_model = hybrid::HybridDavinciModel::Create(ge_root_model); GE_CHECK_NOTNULL(hybrid_model); hybrid_model->SetListener(listener); hybrid_model->SetModelId(model_id); hybrid_model->SetDeviceId(GetContext().DeviceId()); + hybrid_model->SetModelName(model_name); GE_CHK_STATUS_RET(hybrid_model->Init(), "Failed to init hybrid model. model_id = %u", model_id); auto shared_model = std::shared_ptr(hybrid_model.release()); InsertModel(model_id, shared_model); @@ -304,10 +306,11 @@ Status ModelManager::LoadModelOnline(uint32_t &model_id, const shared_ptrCheckIsUnknownShape(is_shape_unknown), "CheckIsUnknownShape failed, model id:%u", model_id); if (is_shape_unknown || GetContext().GetHostExecFlag()) { - return DoLoadHybridModelOnline(model_id, ge_root_model, listener); + return DoLoadHybridModelOnline(model_id, model_name, ge_root_model, listener); } mmTimespec timespec = mmGetTickCount(); @@ -321,7 +324,7 @@ Status ModelManager::LoadModelOnline(uint32_t &model_id, const shared_ptrSetId(model_id); davinci_model->SetDeviceId(GetContext().DeviceId()); - const DumpProperties &dump_properties = PropertiesManager::Instance().GetDumpProperties(GetContext().SessionId()); + const DumpProperties &dump_properties = DumpManager::GetInstance().GetDumpProperties(GetContext().SessionId()); davinci_model->SetDumpProperties(dump_properties); dump_properties_ = dump_properties; @@ -1036,7 +1039,7 @@ Status ModelManager::GenSessionId(uint64_t &session_id) { Status ModelManager::LoadModelOffline(uint32_t &model_id, const ModelData &model, shared_ptr listener, void *dev_ptr, size_t mem_size, void *weight_ptr, size_t weight_size) { GE_CHK_BOOL_RET_STATUS(model.key.empty() || mmAccess2(model.key.c_str(), M_F_OK) == EN_OK, - ACL_ERROR_GE_PARAM_INVALID, "input key file path %s is invalid, %s", model.key.c_str(), strerror(errno)); + ACL_ERROR_GE_PARAM_INVALID, "Input key file path %s is invalid, %s", model.key.c_str(), strerror(errno)); GenModelId(&model_id); mmTimespec timespec = mmGetTickCount(); @@ -1053,7 +1056,7 @@ Status ModelManager::LoadModelOffline(uint32_t &model_id, const ModelData &model GE_CHK_STATUS_RET(model_helper.GetGeRootModel()->CheckIsUnknownShape(is_shape_unknown), "CheckIsUnknownShape failed, model id:%u", model_id); if (is_shape_unknown || GetContext().GetHostExecFlag()) { - return DoLoadHybridModelOnline(model_id, model_helper.GetGeRootModel(), listener); + return DoLoadHybridModelOnline(model_id, model.om_name, model_helper.GetGeRootModel(), listener); } } @@ -1081,8 +1084,8 @@ Status ModelManager::LoadModelOffline(uint32_t &model_id, const ModelData &model } davinci_model->SetDeviceId(device_id); davinci_model->SetOmName(model.om_name); - if (DumpManager::GetInstance().GetDumpProperties().IsDumpOpen()) { - davinci_model->SetDumpProperties(DumpManager::GetInstance().GetDumpProperties()); + if (DumpManager::GetInstance().GetDumpProperties(kInferSessionId).IsDumpOpen()) { + davinci_model->SetDumpProperties(DumpManager::GetInstance().GetDumpProperties(kInferSessionId)); } else { davinci_model->SetDumpProperties(dump_properties_); } @@ -1092,9 +1095,9 @@ Status ModelManager::LoadModelOffline(uint32_t &model_id, const ModelData &model /// Update session_id for infer in load model to avoid the same session_id. uint64_t new_session_id; ret = GenSessionId(new_session_id); - GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, break, "Generate session_id for infer failed."); + GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, break, "Generate session_id for inference failed."); ret = davinci_model->UpdateSessionId(new_session_id); - GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, break, "Update session_id for infer failed."); + GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, break, "Update session_id for inference failed."); ret = davinci_model->Init(dev_ptr, mem_size, weight_ptr, weight_size); GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, break, "DavinciInit failed."); diff --git a/ge/graph/load/model_manager/model_manager.h b/ge/graph/load/model_manager/model_manager.h index 8aa09418..00d8958f 100755 --- a/ge/graph/load/model_manager/model_manager.h +++ b/ge/graph/load/model_manager/model_manager.h @@ -73,7 +73,7 @@ class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ModelManager { ge::Status LoadModelOnline(uint32_t &model_id, const std::shared_ptr &ge_root_model, std::shared_ptr listener); - ge::Status DoLoadHybridModelOnline(uint32_t model_id, const shared_ptr &ge_root_model, + ge::Status DoLoadHybridModelOnline(uint32_t model_id, const string &model_name, const shared_ptr &ge_root_model, const std::shared_ptr &listener); /// diff --git a/ge/graph/manager/graph_manager.cc b/ge/graph/manager/graph_manager.cc index 8cff22ae..1ec067f1 100755 --- a/ge/graph/manager/graph_manager.cc +++ b/ge/graph/manager/graph_manager.cc @@ -26,6 +26,7 @@ #include "common/math/math_util.h" #include "common/thread_pool.h" +#include "common/dump/dump_manager.h" #include "analyzer/analyzer.h" #include "graph/common/ge_call_wrapper.h" #include "graph/common/local_context.h" @@ -3115,7 +3116,7 @@ Status GraphManager::Build(const GraphNodePtr &graph_node, ComputeGraphPtr &comp } bool is_always_dump = false; - if (!PropertiesManager::Instance().GetDumpProperties(session_id).GetDumpPath().empty()) { + if (!DumpManager::GetInstance().GetDumpProperties(session_id).GetDumpPath().empty()) { is_always_dump = true; } diff --git a/ge/hybrid/executor/hybrid_model_async_executor.cc b/ge/hybrid/executor/hybrid_model_async_executor.cc index 97fb9d50..c508cc4b 100644 --- a/ge/hybrid/executor/hybrid_model_async_executor.cc +++ b/ge/hybrid/executor/hybrid_model_async_executor.cc @@ -46,6 +46,10 @@ void HybridModelAsyncExecutor::SetModelId(uint32_t model_id) { model_id_ = model_id; } +void HybridModelAsyncExecutor::SetModelName(const string &model_name) { + om_name_ = model_name; +} + Status HybridModelAsyncExecutor::EnqueueData(const shared_ptr &data) { GE_CHK_STATUS_EXEC(data_inputer_->Push(data), return domi::DATA_QUEUE_ISFULL, "Data queue is full, please call again later, model_id %u ", model_id_); diff --git a/ge/hybrid/executor/hybrid_model_async_executor.h b/ge/hybrid/executor/hybrid_model_async_executor.h index dec7e384..4790248b 100644 --- a/ge/hybrid/executor/hybrid_model_async_executor.h +++ b/ge/hybrid/executor/hybrid_model_async_executor.h @@ -49,6 +49,8 @@ class HybridModelAsyncExecutor { void SetModelId(uint32_t model_id); + void SetModelName(const string &model_name); + Status Stop(); Status EnqueueData(const std::shared_ptr &data); @@ -91,6 +93,7 @@ class HybridModelAsyncExecutor { std::map input_tensor_desc_; std::vector is_input_dynamic_; std::shared_ptr listener_; + string om_name_; }; } // namespace hybrid } // namespace ge diff --git a/ge/hybrid/executor/hybrid_model_executor.cc b/ge/hybrid/executor/hybrid_model_executor.cc index c4154abb..80b8983a 100755 --- a/ge/hybrid/executor/hybrid_model_executor.cc +++ b/ge/hybrid/executor/hybrid_model_executor.cc @@ -17,6 +17,7 @@ #include "hybrid_model_executor.h" #include "graph/ge_context.h" #include "graph/runtime_inference_context.h" +#include "common/dump/dump_manager.h" namespace ge { namespace hybrid { @@ -107,7 +108,7 @@ Status HybridModelExecutor::InitExecutionContext() { GE_CHECK_NOTNULL(context_.allocator); context_.callback_manager = std::unique_ptr(new(std::nothrow)CallbackManager()); GE_CHECK_NOTNULL(context_.callback_manager); - context_.dump_properties = PropertiesManager::Instance().GetDumpProperties(context_.session_id); + context_.dump_properties = DumpManager::GetInstance().GetDumpProperties(context_.session_id); const char *profiling_level = std::getenv(kEnvProfilingLevel); if (profiling_level != nullptr) { context_.profiling_level = std::strtol(profiling_level, nullptr, kIntBase); diff --git a/ge/hybrid/executor/hybrid_model_pipeline_executor.cc b/ge/hybrid/executor/hybrid_model_pipeline_executor.cc index 6c824bf8..4706fa97 100644 --- a/ge/hybrid/executor/hybrid_model_pipeline_executor.cc +++ b/ge/hybrid/executor/hybrid_model_pipeline_executor.cc @@ -1,6 +1,7 @@ #include "hybrid_model_pipeline_executor.h" #include "common/math/math_util.h" +#include "common/dump/dump_manager.h" #include "graph/ge_context.h" #include "graph/runtime_inference_context.h" @@ -145,7 +146,7 @@ Status StageExecutor::InitExecutionContext() { GE_CHECK_NOTNULL(context_.allocator); context_.callback_manager = std::unique_ptr(new (std::nothrow) CallbackManager()); GE_CHECK_NOTNULL(context_.callback_manager); - context_.dump_properties = PropertiesManager::Instance().GetDumpProperties(context_.session_id); + context_.dump_properties = DumpManager::GetInstance().GetDumpProperties(context_.session_id); if (IsLogEnable(GE_MODULE_NAME, DLOG_DEBUG)) { context_.trace_enabled = true; } diff --git a/ge/hybrid/hybrid_davinci_model.cc b/ge/hybrid/hybrid_davinci_model.cc index 7009331c..430dfa85 100755 --- a/ge/hybrid/hybrid_davinci_model.cc +++ b/ge/hybrid/hybrid_davinci_model.cc @@ -76,6 +76,11 @@ class HybridDavinciModel::Impl { executor_.SetDeviceId(device_id); } + void SetModelName(const string &model_name) { + model_.SetModelName(model_name); + executor_.SetModelName(model_name); + } + uint64_t GetSessionId() { return model_.GetSessionId(); } @@ -176,6 +181,12 @@ void HybridDavinciModel::SetDeviceId(uint32_t device_id) { } } +void HybridDavinciModel::SetModelName(const string &model_name) { + if (impl_ != nullptr) { + impl_->SetModelName(model_name); + } +} + Status HybridDavinciModel::GetDynamicBatchInfo(std::vector> &batch_info, int32_t &dynamic_type) { GE_CHECK_NOTNULL(impl_); return impl_->GetDynamicBatchInfo(batch_info, dynamic_type); diff --git a/ge/hybrid/hybrid_davinci_model.h b/ge/hybrid/hybrid_davinci_model.h index 369c732a..74dca9ed 100644 --- a/ge/hybrid/hybrid_davinci_model.h +++ b/ge/hybrid/hybrid_davinci_model.h @@ -57,6 +57,8 @@ class HybridDavinciModel { void SetDeviceId(uint32_t device_id); + void SetModelName(const string &model_name); + uint64_t GetSessionId(); Status GetDynamicBatchInfo(std::vector> &batch_info, int32_t &dynamic_type); diff --git a/ge/hybrid/hybrid_davinci_model_stub.cc b/ge/hybrid/hybrid_davinci_model_stub.cc index 366845c5..5b10fb7a 100644 --- a/ge/hybrid/hybrid_davinci_model_stub.cc +++ b/ge/hybrid/hybrid_davinci_model_stub.cc @@ -61,6 +61,9 @@ void HybridDavinciModel::SetModelId(uint32_t model_id) { void HybridDavinciModel::SetDeviceId(uint32_t device_id) { } +void HybridDavinciModel::SetModelName(const string &model_name) { +} + uint64_t HybridDavinciModel::GetSessionId() { return 0; } diff --git a/ge/hybrid/model/hybrid_model.h b/ge/hybrid/model/hybrid_model.h index 1f973d1e..500f0472 100644 --- a/ge/hybrid/model/hybrid_model.h +++ b/ge/hybrid/model/hybrid_model.h @@ -65,6 +65,14 @@ class HybridModel { model_id_ = model_id; } + void SetModelName(const string &model_name) { + om_name_ = model_name; + } + + const std::string &GetOmName() const { + return om_name_; + } + uint32_t GetModelId() const { return model_id_; } @@ -143,6 +151,7 @@ class HybridModel { uint8_t *var_mem_base_ = nullptr; std::unique_ptr weight_buffer_; RuntimeParam root_runtime_param_; + string om_name_; }; } // namespace hybrid } // namespace ge diff --git a/ge/hybrid/node_executor/compiledsubgraph/known_node_executor.cc b/ge/hybrid/node_executor/compiledsubgraph/known_node_executor.cc index 0837ffff..1d6e814b 100755 --- a/ge/hybrid/node_executor/compiledsubgraph/known_node_executor.cc +++ b/ge/hybrid/node_executor/compiledsubgraph/known_node_executor.cc @@ -179,6 +179,7 @@ Status KnownNodeExecutor::LoadTask(const HybridModel &model, const NodePtr &node // set known node flag as true davinci_model->SetKnownNode(true); davinci_model->SetId(model.GetModelId()); + davinci_model->SetOmName(model.GetOmName()); // set model id as root node's node id davinci_model->SetSubModelId(node->GetOpDesc()->GetId()); GELOGD("KnownNodeExecutor::LoadTask node id %ld.", node->GetOpDesc()->GetId()); diff --git a/ge/session/inner_session.cc b/ge/session/inner_session.cc index 6a56fc05..d11ba10e 100755 --- a/ge/session/inner_session.cc +++ b/ge/session/inner_session.cc @@ -23,6 +23,7 @@ #include "analyzer/analyzer.h" #include "adx_datadump_server.h" #include "common/dump/dump_properties.h" +#include "common/dump/dump_manager.h" #include "common/util.h" #include "framework/common/debug/ge_log.h" #include "graph/ge_context.h" @@ -374,13 +375,13 @@ Status InnerSession::AddDumpProperties(const DumpProperties &dump_properties) { is_dump_server_inited_ = true; } } - PropertiesManager::Instance().AddDumpProperties(session_id_, dump_properties); + DumpManager::GetInstance().AddDumpProperties(session_id_, dump_properties); return SUCCESS; } Status InnerSession::RemoveDumpProperties() { - PropertiesManager::Instance().RemoveDumpProperties(session_id_); - if (is_dump_server_inited_ && PropertiesManager::Instance().GetDumpPropertiesMap().empty()) { + DumpManager::GetInstance().RemoveDumpProperties(session_id_); + if (is_dump_server_inited_ && DumpManager::GetInstance().GetDumpPropertiesMap().empty()) { GE_IF_BOOL_EXEC(AdxDataDumpServerUnInit() != kDumpStatus, GELOGE(PARAM_INVALID, "Data dump server uninit failed"); return PARAM_INVALID) GELOGI("UnInit adx data dump server success"); diff --git a/ge/single_op/single_op.cc b/ge/single_op/single_op.cc index 168ca2c5..a6069706 100755 --- a/ge/single_op/single_op.cc +++ b/ge/single_op/single_op.cc @@ -199,6 +199,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status SingleOp::ExecuteAsync(c if (ret != SUCCESS) { return ret; } + GE_CHK_STATUS_RET(task->OpenDump(stream_), "Open single op %s dump filed",task->GetOpdesc()->GetName().c_str()); GE_CHK_STATUS_RET_NOLOG(ProfilingTaskInfo(task, kShapeTypeStatic)); } @@ -279,6 +280,7 @@ Status DynamicSingleOp::ExecuteAsync(const vector &input_desc, GE_CHECK_NOTNULL(op_task_); GE_CHK_STATUS_RET_NOLOG(op_task_->LaunchKernel(input_desc, input_buffers, output_desc, output_buffers, stream_)); + GE_CHK_STATUS_RET_NOLOG(op_task_->OpenDump(stream_)); GE_CHK_STATUS_RET_NOLOG(ProfilingTaskInfo(op_task_.get(), kShapeTypeDynamic)); return SUCCESS; } diff --git a/ge/single_op/task/op_task.cc b/ge/single_op/task/op_task.cc index df4161c7..266176ab 100755 --- a/ge/single_op/task/op_task.cc +++ b/ge/single_op/task/op_task.cc @@ -36,6 +36,7 @@ constexpr int kLaunchRetryTimes = 1000; constexpr int kSleepTime = 10; constexpr uint64_t kReleaseFlag = 1; constexpr int kCopyNum = 2; +constexpr uint64_t kInferSessionId = 0; void FreeHbm(void *var) { if (var) { (void)rtFree(var); @@ -44,7 +45,7 @@ void FreeHbm(void *var) { } // namespace Status OpTask::OpenDump(rtStream_t stream) { - if (DumpManager::GetInstance().GetDumpProperties().IsSingleOpNeedDump()) { + if (DumpManager::GetInstance().GetDumpProperties(kInferSessionId).IsSingleOpNeedDump()) { GELOGI("Dump is open in single op, start to set dump info"); std::vector input_addrs; std::vector output_adds; @@ -68,7 +69,7 @@ Status OpTask::OpenDump(rtStream_t stream) { uint64_t output_addr = arg_base[input_size + j]; output_adds.emplace_back(output_addr); } - dump_op_.SetDumpInfo(DumpManager::GetInstance().GetDumpProperties(), op_desc_, input_addrs, output_adds, stream); + dump_op_.SetDumpInfo(DumpManager::GetInstance().GetDumpProperties(kInferSessionId), op_desc_, input_addrs, output_adds, stream); auto status = dump_op_.LaunchDumpOp(); if (status != SUCCESS) { GELOGE(status, "Launch dump op failed in single op"); @@ -194,11 +195,6 @@ Status TbeOpTask::LaunchKernel(rtStream_t stream) { return RT_ERROR_TO_GE_STATUS(ret); } GELOGI("[TASK_INFO] %s", this->stub_name_.c_str()); - auto status = OpenDump(stream); - if (status != SUCCESS) { - GELOGE(status, "Open dump failed in the tbe single op %s", this->stub_name_.c_str()); - return status; - } return SUCCESS; } @@ -491,6 +487,10 @@ Status AiCpuBaseTask::UpdateOutputShape(vector &output_desc) { aicpu_ext_handle_->GetOutputShapeAndType(i, shape, data_type); GE_CHK_STATUS_RET(UpdateShapeToOutputDesc(shape, output_desc[i]), "AiCpuCCTask Update [%zu]th output shape failed.", i); + if (DumpManager::GetInstance().GetDumpProperties(kInferSessionId).IsSingleOpNeedDump()) { + GE_CHK_STATUS_RET(op_desc_->UpdateOutputDesc(i, output_desc[i]), + "AiCpuCCTask Update [%zu]th output desc failed.", i); + } } GELOGD("Update DEPEND_SHAPE_RANGE AiCpuBaseTask outputshape finished."); return SUCCESS; @@ -601,12 +601,6 @@ Status AiCpuTask::LaunchKernel(rtStream_t stream) { } GELOGI("[TASK_INFO] %lu/%s", kernel_id_, op_type_.c_str()); - auto status = OpenDump(stream); - if (status != SUCCESS) { - GELOGE(status, "Open dump failed in aicpu single op %s", this->op_type_.c_str()); - return status; - } - GELOGD("Done launch kernel successfully. task = %s", this->op_type_.c_str()); return SUCCESS; } @@ -700,6 +694,10 @@ Status AiCpuTask::UpdateShapeByHbmBuffer(vector &output_desc) { GE_CHK_STATUS_RET(UpdateShapeToOutputDesc(GeShape(shape_dims), output_desc[i]), "AiCpuTask update [%zu]th output shape failed.", i); + if (DumpManager::GetInstance().GetDumpProperties(kInferSessionId).IsSingleOpNeedDump()) { + GE_CHK_STATUS_RET(op_desc_->UpdateOutputDesc(i, output_desc[i]), + "AiCpuTask update [%zu]th output desc failed.", i); + } } return SUCCESS; } @@ -876,12 +874,6 @@ Status AiCpuCCTask::LaunchKernel(rtStream_t stream) { } GELOGI("[TASK_INFO] %lu/%s", kernel_id_, op_type_.c_str()); GELOGD("Invoke rtCpuKernelLaunch succeeded"); - auto status = OpenDump(stream); - if (status != SUCCESS) { - GELOGE(status, "Open dump failed in the aicpucc single op %s", this->kernel_name_.c_str()); - return status; - } - return SUCCESS; } diff --git a/tests/ut/ge/CMakeLists.txt b/tests/ut/ge/CMakeLists.txt index 688e393c..f19560dc 100755 --- a/tests/ut/ge/CMakeLists.txt +++ b/tests/ut/ge/CMakeLists.txt @@ -354,6 +354,7 @@ set(COMMON_FORMAT_SRC_FILES "${GE_CODE_DIR}/ge/common/formats/format_transfers/format_transfer_fracz_hwcn.cc" "${GE_CODE_DIR}/ge/common/formats/utils/formats_trans_utils.cc" "${GE_CODE_DIR}/ge/graph/manager/util/hcom_util.cc" + "${GE_CODE_DIR}/ge/common/dump/dump_manager.cc" ) set(GRAPH_OPTIMIZE_COMMON_SRC_FILES @@ -730,6 +731,7 @@ set(MULTI_PARTS_TEST_FILES "graph_ir/ge_operator_factory_unittest.cc" "graph/transop_util_unittest.cc" "common/datatype_transfer_unittest.cc" + "common/dump_manager_unittest.cc" "common/format_transfer_unittest.cc" "common/format_transfer_transpose_unittest.cc" "common/format_transfer_nchw_5d_unittest.cc" diff --git a/tests/ut/ge/common/dump_manager_unittest.cc b/tests/ut/ge/common/dump_manager_unittest.cc new file mode 100644 index 00000000..7f3880f2 --- /dev/null +++ b/tests/ut/ge/common/dump_manager_unittest.cc @@ -0,0 +1,76 @@ +/** + * Copyright 2019-2020 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "common/dump/dump_manager.h" +#include "common/debug/log.h" +#include "common/ge_inner_error_codes.h" + +namespace ge { +class UTEST_dump_manager : public testing::Test { + protected: + void SetUp() {} + void TearDown() {} +}; + TEST_F(UTEST_dump_manager, is_dump_open_success) { + DumpConfig dump_config; + dump_config.dump_path = "/test"; + dump_config.dump_mode = "all"; + dump_config.dump_status = "on"; + dump_config.dump_op_switch = "on"; + auto ret = DumpManager::GetInstance().SetDumpConf(dump_config); + auto dump = DumpManager::GetInstance().GetDumpProperties(0); + bool result = dump.IsDumpOpen(); + dump.ClearDumpInfo(); + EXPECT_EQ(result, true); + } + + TEST_F(UTEST_dump_manager, is_dump_op_success) { + DumpConfig dump_config; + dump_config.dump_path = "/test"; + dump_config.dump_mode = "all"; + dump_config.dump_status = "off"; + auto ret = DumpManager::GetInstance().SetDumpConf(dump_config); + EXPECT_EQ(ret, ge::SUCCESS); + } + +TEST_F(UTEST_dump_manager, is_dump_single_op_close_success) { + DumpConfig dump_config; + dump_config.dump_path = "/test"; + dump_config.dump_mode = "all"; + dump_config.dump_status = "on"; + dump_config.dump_op_switch = "off"; + auto ret = DumpManager::GetInstance().SetDumpConf(dump_config); + EXPECT_EQ(ret, ge::PARAM_INVALID); + } + + TEST_F(UTEST_dump_manager, dump_status_empty) { + DumpConfig dump_config; + dump_config.dump_path = "/test"; + dump_config.dump_mode = "all"; + dump_config.dump_op_switch = "off"; + auto ret = DumpManager::GetInstance().SetDumpConf(dump_config); + EXPECT_EQ(ret, ge::SUCCESS); + } + + TEST_F(UTEST_dump_manager, add_dump_properties_success) { + DumpProperties dump_properties; + DumpManager::GetInstance().AddDumpProperties(0, dump_properties); + auto dump = DumpManager::GetInstance().GetDumpProperties(0); + DumpManager::GetInstance().RemoveDumpProperties(0); + } +} // namespace ge \ No newline at end of file