Browse Source

infer dump

tags/v1.2.0
zhou_chao1993 3 years ago
parent
commit
20daa5638b
25 changed files with 188 additions and 90 deletions
  1. +27
    -17
      ge/common/dump/dump_manager.cc
  2. +5
    -5
      ge/common/dump/dump_manager.h
  3. +5
    -1
      ge/common/dump/dump_properties.cc
  4. +1
    -1
      ge/common/dump/dump_properties.h
  5. +0
    -20
      ge/common/properties_manager.cc
  6. +0
    -9
      ge/common/properties_manager.h
  7. +2
    -1
      ge/graph/build/model_builder.cc
  8. +1
    -1
      ge/graph/load/model_manager/davinci_model.h
  9. +12
    -9
      ge/graph/load/model_manager/model_manager.cc
  10. +1
    -1
      ge/graph/load/model_manager/model_manager.h
  11. +2
    -1
      ge/graph/manager/graph_manager.cc
  12. +4
    -0
      ge/hybrid/executor/hybrid_model_async_executor.cc
  13. +3
    -0
      ge/hybrid/executor/hybrid_model_async_executor.h
  14. +2
    -1
      ge/hybrid/executor/hybrid_model_executor.cc
  15. +2
    -1
      ge/hybrid/executor/hybrid_model_pipeline_executor.cc
  16. +11
    -0
      ge/hybrid/hybrid_davinci_model.cc
  17. +2
    -0
      ge/hybrid/hybrid_davinci_model.h
  18. +3
    -0
      ge/hybrid/hybrid_davinci_model_stub.cc
  19. +9
    -0
      ge/hybrid/model/hybrid_model.h
  20. +1
    -0
      ge/hybrid/node_executor/compiledsubgraph/known_node_executor.cc
  21. +4
    -3
      ge/session/inner_session.cc
  22. +2
    -0
      ge/single_op/single_op.cc
  23. +11
    -19
      ge/single_op/task/op_task.cc
  24. +2
    -0
      tests/ut/ge/CMakeLists.txt
  25. +76
    -0
      tests/ut/ge/common/dump_manager_unittest.cc

+ 27
- 17
ge/common/dump/dump_manager.cc View File

@@ -22,6 +22,7 @@ namespace {
const char *const kDumpOFF = "OFF";
const char *const kDumpoff = "off";
const char *const kDumpOn = "on";
const uint64_t kInferSessionId = 0;
} // namespace
namespace ge {
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY DumpManager &DumpManager::GetInstance() {
@@ -30,15 +31,14 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY DumpManager &DumpManager::GetIn
}

FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status DumpManager::SetDumpConf(const DumpConfig &dump_config) {
std::lock_guard<std::mutex> lock(mutex_);
dump_properties_.ClearDumpPropertyValue();
dump_properties_.ClearDumpInfo();
DumpProperties dump_properties;
std::string dump_status;
std::string dump_path;
std::string dump_mode;
std::string dump_op_switch;

if (dump_config.dump_status.empty()) {
dump_properties_map_.emplace(kInferSessionId, dump_properties);
GELOGI("Dump does not open");
return SUCCESS;
}
@@ -46,14 +46,16 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status DumpManager::SetDumpConf
dump_status = dump_config.dump_status;
GELOGI("Dump status is %s", dump_status.c_str());
if (dump_config.dump_status == kDumpoff || dump_config.dump_status == kDumpOFF) {
dump_properties_.ClearDumpPropertyValue();
dump_properties.ClearDumpPropertyValue();
dump_properties_map_.emplace(kInferSessionId, dump_properties);
return SUCCESS;
}
dump_properties_.SetDumpStatus(dump_status);
dump_properties.SetDumpStatus(dump_status);

dump_op_switch = dump_config.dump_op_switch;
dump_properties_.SetDumpOpSwitch(dump_op_switch);
dump_properties.SetDumpOpSwitch(dump_op_switch);
if (dump_op_switch == kDumpoff && dump_config.dump_list.empty()) {
dump_properties_map_.emplace(kInferSessionId, dump_properties);
GELOGE(PARAM_INVALID, "Dump list is invalid,dump_op_switch is %s", dump_op_switch.c_str());
return PARAM_INVALID;
}
@@ -67,15 +69,15 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status DumpManager::SetDumpConf
GELOGI("Dump layer is %s in model", layer.c_str());
dump_layers.insert(layer);
}
dump_properties_.AddPropertyValue(model_name, dump_layers);
dump_properties.AddPropertyValue(model_name, dump_layers);
}
if (dump_op_switch == kDumpOn) {
GELOGI("Start to dump model and single op,dumo op switch is %s", dump_op_switch.c_str());
GELOGI("Start to dump model and single op,dump op switch is %s", dump_op_switch.c_str());
} else {
GELOGI("Only dump model,dump op switch is %s", dump_op_switch.c_str());
}
} else {
GELOGI("Only dump single op,dumo op switch is %s", dump_op_switch.c_str());
GELOGI("Only dump single op,dump op switch is %s", dump_op_switch.c_str());
}

dump_path = dump_config.dump_path;
@@ -89,27 +91,35 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status DumpManager::SetDumpConf
}
dump_path = dump_path + CurrentTimeInStr() + "/";
GELOGI("Dump path is %s", dump_path.c_str());
dump_properties_.SetDumpPath(dump_path);
dump_properties.SetDumpPath(dump_path);

dump_mode = dump_config.dump_mode;
GELOGI("Dump mode is %s", dump_mode.c_str());
dump_properties_.SetDumpMode(dump_mode);
dump_properties.SetDumpMode(dump_mode);
dump_properties_map_.emplace(kInferSessionId, dump_properties);

return SUCCESS;
}

FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY const DumpProperties &DumpManager::GetDumpProperties() {
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY const DumpProperties &DumpManager::GetDumpProperties(
uint64_t session_id) {
std::lock_guard<std::mutex> lock(mutex_);
return dump_properties_;
// If session_id is not found in dump_properties_map_, operator[] will insert one.
return dump_properties_map_[session_id];
}

FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void DumpManager::SetModelName(const std::string &model_name) {
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void DumpManager::AddDumpProperties(
uint64_t session_id, const DumpProperties &dump_properties) {
std::lock_guard<std::mutex> lock(mutex_);
model_name_ = model_name;
dump_properties_map_.emplace(session_id, dump_properties);
}

FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY const std::string &DumpManager::GetModelName() {
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void DumpManager::RemoveDumpProperties(uint64_t session_id) {
std::lock_guard<std::mutex> lock(mutex_);
return model_name_;
auto iter = dump_properties_map_.find(session_id);
if (iter != dump_properties_map_.end()) {
dump_properties_map_.erase(iter);
}
}

} // namespace ge

+ 5
- 5
ge/common/dump/dump_manager.h View File

@@ -28,14 +28,14 @@ class DumpManager {
static DumpManager &GetInstance();

Status SetDumpConf(const DumpConfig &dump_config);
const DumpProperties &GetDumpProperties();
void SetModelName(const std::string &model_name);
const std::string &GetModelName();
const DumpProperties &GetDumpProperties(uint64_t session_id);
const std::map<uint64_t, DumpProperties> &GetDumpPropertiesMap() { return dump_properties_map_; }
void AddDumpProperties(uint64_t session_id, const DumpProperties &dump_properties);
void RemoveDumpProperties(uint64_t session_id);

private:
DumpProperties dump_properties_;
std::mutex mutex_;
std::string model_name_;
std::map<uint64_t, DumpProperties> dump_properties_map_;
};
} // namespace ge
#endif // GE_COMMON_DUMP_DUMP_MANAGER_H_

+ 5
- 1
ge/common/dump/dump_properties.cc View File

@@ -122,6 +122,8 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void DumpProperties::ClearDumpI
dump_path_.clear();
dump_step_.clear();
dump_mode_.clear();
dump_op_switch_.clear();
dump_status_.clear();
is_op_debug_ = false;
op_debug_mode_ = 0;
}
@@ -201,7 +203,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY const std::string &DumpProperti
}

FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void DumpProperties::SetDumpOpSwitch(
const std::string dump_op_switch) {
const std::string &dump_op_switch) {
dump_op_switch_ = dump_op_switch;
}

@@ -230,6 +232,8 @@ void DumpProperties::CopyFrom(const DumpProperties &other) {
dump_path_ = other.dump_path_;
dump_step_ = other.dump_step_;
dump_mode_ = other.dump_mode_;
dump_status_ = other.dump_status_;
dump_op_switch_ = other.dump_op_switch_;

model_dump_properties_map_ = other.model_dump_properties_map_;
is_op_debug_ = other.is_op_debug_;


+ 1
- 1
ge/common/dump/dump_properties.h View File

@@ -65,7 +65,7 @@ class DumpProperties {

const std::string &GetDumpStatus() const;

void SetDumpOpSwitch(const std::string dump_op_switch);
void SetDumpOpSwitch(const std::string &dump_op_switch);

const std::string &GetDumpOpSwitch() const;



+ 0
- 20
ge/common/properties_manager.cc View File

@@ -165,24 +165,4 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void PropertiesManager::SetProp
delimiter = de;
}

FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY DumpProperties &PropertiesManager::GetDumpProperties(
uint64_t session_id) {
std::lock_guard<std::mutex> lock(mutex_);
// If session_id is not found in dump_properties_map_, operator[] will insert one.
return dump_properties_map_[session_id];
}

FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void PropertiesManager::AddDumpProperties(
uint64_t session_id, const DumpProperties &dump_properties) {
std::lock_guard<std::mutex> lock(mutex_);
dump_properties_map_.emplace(session_id, dump_properties);
}

FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void PropertiesManager::RemoveDumpProperties(uint64_t session_id) {
std::lock_guard<std::mutex> lock(mutex_);
auto iter = dump_properties_map_.find(session_id);
if (iter != dump_properties_map_.end()) {
dump_properties_map_.erase(iter);
}
}
} // namespace ge

+ 0
- 9
ge/common/properties_manager.h View File

@@ -83,13 +83,6 @@ class PropertiesManager {
*/
void SetPropertyDelimiter(const std::string &de);

DumpProperties &GetDumpProperties(uint64_t session_id);

const map<uint64_t, DumpProperties> &GetDumpPropertiesMap() { return dump_properties_map_; }

void AddDumpProperties(uint64_t session_id, const DumpProperties &dump_properties);
void RemoveDumpProperties(uint64_t session_id);

private:
// Private construct, destructor
PropertiesManager();
@@ -111,8 +104,6 @@ class PropertiesManager {

std::map<std::string, std::string> properties_map_;
std::mutex mutex_;

std::map<uint64_t, DumpProperties> dump_properties_map_;
};
} // namespace ge



+ 2
- 1
ge/graph/build/model_builder.cc View File

@@ -19,6 +19,7 @@
#include <set>
#include <unordered_map>
#include "common/ge/ge_util.h"
#include "common/dump/dump_manager.h"
#include "framework/common/debug/ge_log.h"
#include "graph/anchor.h"
#include "graph/attr_value.h"
@@ -429,7 +430,7 @@ Status ModelBuilder::BuildModelDef(ge::Model &model) {
GE_CHK_BOOL_EXEC(ge::AttrUtils::SetBool(&model, ATTR_NAME_SWITCH_FOR_L1_FUSION, is_l1_fusion_enable_),
GELOGE(FAILED, "SetBool of ATTR_NAME_SWITCH_FOR_L1_FUSION failed.");
return FAILED);
const DumpProperties &dump_properties = PropertiesManager::Instance().GetDumpProperties(session_id_);
const DumpProperties &dump_properties = DumpManager::GetInstance().GetDumpProperties(session_id_);
bool is_op_debug = dump_properties.IsOpDebugOpen();
if (is_op_debug) {
if (!ge::AttrUtils::SetBool(&model, ATTR_OP_DEBUG_FLAG, is_op_debug)) {


+ 1
- 1
ge/graph/load/model_manager/davinci_model.h View File

@@ -536,7 +536,7 @@ class DavinciModel {
vector<InputOutputDims> &output_dims) const;

// om file name
void SetOmName(string om_name) { om_name_ = om_name; }
void SetOmName(const string &om_name) { om_name_ = om_name; }

void SetDumpProperties(const DumpProperties &dump_properties) { data_dumper_.SetDumpProperties(dump_properties); }
const DumpProperties &GetDumpProperties() const { return data_dumper_.GetDumpProperties(); }


+ 12
- 9
ge/graph/load/model_manager/model_manager.cc View File

@@ -55,6 +55,7 @@ const char *const kDeleteCustOp = "deleteCustOp";
const int kTimeSpecNano = 1000000000;
const int kTimeSpecMiro = 1000000;
const int kOpNameMaxSize = 100;
const uint64_t kInferSessionId = 0;
#pragma pack(push, 1)
struct CustAicpuSoBuf {
uint64_t kernelSoBuf;
@@ -278,13 +279,14 @@ ge::Status ModelManager::SetDynamicSize(uint32_t model_id, const std::vector<uin
return SUCCESS;
}

ge::Status ModelManager::DoLoadHybridModelOnline(uint32_t model_id, const shared_ptr<ge::GeRootModel> &ge_root_model,
ge::Status ModelManager::DoLoadHybridModelOnline(uint32_t model_id, const string &model_name, const shared_ptr<ge::GeRootModel> &ge_root_model,
const shared_ptr<ModelListener> &listener) {
auto hybrid_model = hybrid::HybridDavinciModel::Create(ge_root_model);
GE_CHECK_NOTNULL(hybrid_model);
hybrid_model->SetListener(listener);
hybrid_model->SetModelId(model_id);
hybrid_model->SetDeviceId(GetContext().DeviceId());
hybrid_model->SetModelName(model_name);
GE_CHK_STATUS_RET(hybrid_model->Init(), "Failed to init hybrid model. model_id = %u", model_id);
auto shared_model = std::shared_ptr<hybrid::HybridDavinciModel>(hybrid_model.release());
InsertModel(model_id, shared_model);
@@ -304,10 +306,11 @@ Status ModelManager::LoadModelOnline(uint32_t &model_id, const shared_ptr<ge::Ge
}

bool is_shape_unknown = false;
string model_name = "";
GE_CHK_STATUS_RET(ge_root_model->CheckIsUnknownShape(is_shape_unknown), "CheckIsUnknownShape failed, model id:%u",
model_id);
if (is_shape_unknown || GetContext().GetHostExecFlag()) {
return DoLoadHybridModelOnline(model_id, ge_root_model, listener);
return DoLoadHybridModelOnline(model_id, model_name, ge_root_model, listener);
}

mmTimespec timespec = mmGetTickCount();
@@ -321,7 +324,7 @@ Status ModelManager::LoadModelOnline(uint32_t &model_id, const shared_ptr<ge::Ge
davinci_model->SetId(model_id);
davinci_model->SetDeviceId(GetContext().DeviceId());

const DumpProperties &dump_properties = PropertiesManager::Instance().GetDumpProperties(GetContext().SessionId());
const DumpProperties &dump_properties = DumpManager::GetInstance().GetDumpProperties(GetContext().SessionId());
davinci_model->SetDumpProperties(dump_properties);
dump_properties_ = dump_properties;

@@ -1036,7 +1039,7 @@ Status ModelManager::GenSessionId(uint64_t &session_id) {
Status ModelManager::LoadModelOffline(uint32_t &model_id, const ModelData &model, shared_ptr<ModelListener> listener,
void *dev_ptr, size_t mem_size, void *weight_ptr, size_t weight_size) {
GE_CHK_BOOL_RET_STATUS(model.key.empty() || mmAccess2(model.key.c_str(), M_F_OK) == EN_OK,
ACL_ERROR_GE_PARAM_INVALID, "input key file path %s is invalid, %s", model.key.c_str(), strerror(errno));
ACL_ERROR_GE_PARAM_INVALID, "Input key file path %s is invalid, %s", model.key.c_str(), strerror(errno));
GenModelId(&model_id);

mmTimespec timespec = mmGetTickCount();
@@ -1053,7 +1056,7 @@ Status ModelManager::LoadModelOffline(uint32_t &model_id, const ModelData &model
GE_CHK_STATUS_RET(model_helper.GetGeRootModel()->CheckIsUnknownShape(is_shape_unknown),
"CheckIsUnknownShape failed, model id:%u", model_id);
if (is_shape_unknown || GetContext().GetHostExecFlag()) {
return DoLoadHybridModelOnline(model_id, model_helper.GetGeRootModel(), listener);
return DoLoadHybridModelOnline(model_id, model.om_name, model_helper.GetGeRootModel(), listener);
}
}

@@ -1081,8 +1084,8 @@ Status ModelManager::LoadModelOffline(uint32_t &model_id, const ModelData &model
}
davinci_model->SetDeviceId(device_id);
davinci_model->SetOmName(model.om_name);
if (DumpManager::GetInstance().GetDumpProperties().IsDumpOpen()) {
davinci_model->SetDumpProperties(DumpManager::GetInstance().GetDumpProperties());
if (DumpManager::GetInstance().GetDumpProperties(kInferSessionId).IsDumpOpen()) {
davinci_model->SetDumpProperties(DumpManager::GetInstance().GetDumpProperties(kInferSessionId));
} else {
davinci_model->SetDumpProperties(dump_properties_);
}
@@ -1092,9 +1095,9 @@ Status ModelManager::LoadModelOffline(uint32_t &model_id, const ModelData &model
/// Update session_id for infer in load model to avoid the same session_id.
uint64_t new_session_id;
ret = GenSessionId(new_session_id);
GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, break, "Generate session_id for infer failed.");
GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, break, "Generate session_id for inference failed.");
ret = davinci_model->UpdateSessionId(new_session_id);
GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, break, "Update session_id for infer failed.");
GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, break, "Update session_id for inference failed.");

ret = davinci_model->Init(dev_ptr, mem_size, weight_ptr, weight_size);
GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(ret != SUCCESS, break, "DavinciInit failed.");


+ 1
- 1
ge/graph/load/model_manager/model_manager.h View File

@@ -73,7 +73,7 @@ class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ModelManager {
ge::Status LoadModelOnline(uint32_t &model_id, const std::shared_ptr<ge::GeRootModel> &ge_root_model,
std::shared_ptr<ModelListener> listener);

ge::Status DoLoadHybridModelOnline(uint32_t model_id, const shared_ptr<ge::GeRootModel> &ge_root_model,
ge::Status DoLoadHybridModelOnline(uint32_t model_id, const string &model_name, const shared_ptr<ge::GeRootModel> &ge_root_model,
const std::shared_ptr<ModelListener> &listener);

///


+ 2
- 1
ge/graph/manager/graph_manager.cc View File

@@ -26,6 +26,7 @@

#include "common/math/math_util.h"
#include "common/thread_pool.h"
#include "common/dump/dump_manager.h"
#include "analyzer/analyzer.h"
#include "graph/common/ge_call_wrapper.h"
#include "graph/common/local_context.h"
@@ -3115,7 +3116,7 @@ Status GraphManager::Build(const GraphNodePtr &graph_node, ComputeGraphPtr &comp
}

bool is_always_dump = false;
if (!PropertiesManager::Instance().GetDumpProperties(session_id).GetDumpPath().empty()) {
if (!DumpManager::GetInstance().GetDumpProperties(session_id).GetDumpPath().empty()) {
is_always_dump = true;
}



+ 4
- 0
ge/hybrid/executor/hybrid_model_async_executor.cc View File

@@ -46,6 +46,10 @@ void HybridModelAsyncExecutor::SetModelId(uint32_t model_id) {
model_id_ = model_id;
}

void HybridModelAsyncExecutor::SetModelName(const string &model_name) {
om_name_ = model_name;
}

Status HybridModelAsyncExecutor::EnqueueData(const shared_ptr<InputDataWrapper> &data) {
GE_CHK_STATUS_EXEC(data_inputer_->Push(data), return domi::DATA_QUEUE_ISFULL,
"Data queue is full, please call again later, model_id %u ", model_id_);


+ 3
- 0
ge/hybrid/executor/hybrid_model_async_executor.h View File

@@ -49,6 +49,8 @@ class HybridModelAsyncExecutor {

void SetModelId(uint32_t model_id);

void SetModelName(const string &model_name);

Status Stop();

Status EnqueueData(const std::shared_ptr<InputDataWrapper> &data);
@@ -91,6 +93,7 @@ class HybridModelAsyncExecutor {
std::map<uint32_t, GeTensorDescPtr> input_tensor_desc_;
std::vector<bool> is_input_dynamic_;
std::shared_ptr<ModelListener> listener_;
string om_name_;
};
} // namespace hybrid
} // namespace ge


+ 2
- 1
ge/hybrid/executor/hybrid_model_executor.cc View File

@@ -17,6 +17,7 @@
#include "hybrid_model_executor.h"
#include "graph/ge_context.h"
#include "graph/runtime_inference_context.h"
#include "common/dump/dump_manager.h"

namespace ge {
namespace hybrid {
@@ -107,7 +108,7 @@ Status HybridModelExecutor::InitExecutionContext() {
GE_CHECK_NOTNULL(context_.allocator);
context_.callback_manager = std::unique_ptr<CallbackManager>(new(std::nothrow)CallbackManager());
GE_CHECK_NOTNULL(context_.callback_manager);
context_.dump_properties = PropertiesManager::Instance().GetDumpProperties(context_.session_id);
context_.dump_properties = DumpManager::GetInstance().GetDumpProperties(context_.session_id);
const char *profiling_level = std::getenv(kEnvProfilingLevel);
if (profiling_level != nullptr) {
context_.profiling_level = std::strtol(profiling_level, nullptr, kIntBase);


+ 2
- 1
ge/hybrid/executor/hybrid_model_pipeline_executor.cc View File

@@ -1,6 +1,7 @@
#include "hybrid_model_pipeline_executor.h"

#include "common/math/math_util.h"
#include "common/dump/dump_manager.h"
#include "graph/ge_context.h"
#include "graph/runtime_inference_context.h"

@@ -145,7 +146,7 @@ Status StageExecutor::InitExecutionContext() {
GE_CHECK_NOTNULL(context_.allocator);
context_.callback_manager = std::unique_ptr<CallbackManager>(new (std::nothrow) CallbackManager());
GE_CHECK_NOTNULL(context_.callback_manager);
context_.dump_properties = PropertiesManager::Instance().GetDumpProperties(context_.session_id);
context_.dump_properties = DumpManager::GetInstance().GetDumpProperties(context_.session_id);
if (IsLogEnable(GE_MODULE_NAME, DLOG_DEBUG)) {
context_.trace_enabled = true;
}


+ 11
- 0
ge/hybrid/hybrid_davinci_model.cc View File

@@ -76,6 +76,11 @@ class HybridDavinciModel::Impl {
executor_.SetDeviceId(device_id);
}

void SetModelName(const string &model_name) {
model_.SetModelName(model_name);
executor_.SetModelName(model_name);
}

uint64_t GetSessionId() {
return model_.GetSessionId();
}
@@ -176,6 +181,12 @@ void HybridDavinciModel::SetDeviceId(uint32_t device_id) {
}
}

void HybridDavinciModel::SetModelName(const string &model_name) {
if (impl_ != nullptr) {
impl_->SetModelName(model_name);
}
}

Status HybridDavinciModel::GetDynamicBatchInfo(std::vector<std::vector<int64_t>> &batch_info, int32_t &dynamic_type) {
GE_CHECK_NOTNULL(impl_);
return impl_->GetDynamicBatchInfo(batch_info, dynamic_type);


+ 2
- 0
ge/hybrid/hybrid_davinci_model.h View File

@@ -57,6 +57,8 @@ class HybridDavinciModel {

void SetDeviceId(uint32_t device_id);

void SetModelName(const string &model_name);

uint64_t GetSessionId();

Status GetDynamicBatchInfo(std::vector<std::vector<int64_t>> &batch_info, int32_t &dynamic_type);


+ 3
- 0
ge/hybrid/hybrid_davinci_model_stub.cc View File

@@ -61,6 +61,9 @@ void HybridDavinciModel::SetModelId(uint32_t model_id) {
void HybridDavinciModel::SetDeviceId(uint32_t device_id) {
}

void HybridDavinciModel::SetModelName(const string &model_name) {
}

uint64_t HybridDavinciModel::GetSessionId() {
return 0;
}


+ 9
- 0
ge/hybrid/model/hybrid_model.h View File

@@ -65,6 +65,14 @@ class HybridModel {
model_id_ = model_id;
}

void SetModelName(const string &model_name) {
om_name_ = model_name;
}

const std::string &GetOmName() const {
return om_name_;
}

uint32_t GetModelId() const {
return model_id_;
}
@@ -143,6 +151,7 @@ class HybridModel {
uint8_t *var_mem_base_ = nullptr;
std::unique_ptr<TensorBuffer> weight_buffer_;
RuntimeParam root_runtime_param_;
string om_name_;
};
} // namespace hybrid
} // namespace ge


+ 1
- 0
ge/hybrid/node_executor/compiledsubgraph/known_node_executor.cc View File

@@ -179,6 +179,7 @@ Status KnownNodeExecutor::LoadTask(const HybridModel &model, const NodePtr &node
// set known node flag as true
davinci_model->SetKnownNode(true);
davinci_model->SetId(model.GetModelId());
davinci_model->SetOmName(model.GetOmName());
// set model id as root node's node id
davinci_model->SetSubModelId(node->GetOpDesc()->GetId());
GELOGD("KnownNodeExecutor::LoadTask node id %ld.", node->GetOpDesc()->GetId());


+ 4
- 3
ge/session/inner_session.cc View File

@@ -23,6 +23,7 @@
#include "analyzer/analyzer.h"
#include "adx_datadump_server.h"
#include "common/dump/dump_properties.h"
#include "common/dump/dump_manager.h"
#include "common/util.h"
#include "framework/common/debug/ge_log.h"
#include "graph/ge_context.h"
@@ -374,13 +375,13 @@ Status InnerSession::AddDumpProperties(const DumpProperties &dump_properties) {
is_dump_server_inited_ = true;
}
}
PropertiesManager::Instance().AddDumpProperties(session_id_, dump_properties);
DumpManager::GetInstance().AddDumpProperties(session_id_, dump_properties);
return SUCCESS;
}

Status InnerSession::RemoveDumpProperties() {
PropertiesManager::Instance().RemoveDumpProperties(session_id_);
if (is_dump_server_inited_ && PropertiesManager::Instance().GetDumpPropertiesMap().empty()) {
DumpManager::GetInstance().RemoveDumpProperties(session_id_);
if (is_dump_server_inited_ && DumpManager::GetInstance().GetDumpPropertiesMap().empty()) {
GE_IF_BOOL_EXEC(AdxDataDumpServerUnInit() != kDumpStatus, GELOGE(PARAM_INVALID, "Data dump server uninit failed");
return PARAM_INVALID)
GELOGI("UnInit adx data dump server success");


+ 2
- 0
ge/single_op/single_op.cc View File

@@ -199,6 +199,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status SingleOp::ExecuteAsync(c
if (ret != SUCCESS) {
return ret;
}
GE_CHK_STATUS_RET(task->OpenDump(stream_), "Open single op %s dump filed",task->GetOpdesc()->GetName().c_str());
GE_CHK_STATUS_RET_NOLOG(ProfilingTaskInfo(task, kShapeTypeStatic));
}

@@ -279,6 +280,7 @@ Status DynamicSingleOp::ExecuteAsync(const vector<GeTensorDesc> &input_desc,
GE_CHECK_NOTNULL(op_task_);

GE_CHK_STATUS_RET_NOLOG(op_task_->LaunchKernel(input_desc, input_buffers, output_desc, output_buffers, stream_));
GE_CHK_STATUS_RET_NOLOG(op_task_->OpenDump(stream_));
GE_CHK_STATUS_RET_NOLOG(ProfilingTaskInfo(op_task_.get(), kShapeTypeDynamic));
return SUCCESS;
}


+ 11
- 19
ge/single_op/task/op_task.cc View File

@@ -36,6 +36,7 @@ constexpr int kLaunchRetryTimes = 1000;
constexpr int kSleepTime = 10;
constexpr uint64_t kReleaseFlag = 1;
constexpr int kCopyNum = 2;
constexpr uint64_t kInferSessionId = 0;
void FreeHbm(void *var) {
if (var) {
(void)rtFree(var);
@@ -44,7 +45,7 @@ void FreeHbm(void *var) {
} // namespace

Status OpTask::OpenDump(rtStream_t stream) {
if (DumpManager::GetInstance().GetDumpProperties().IsSingleOpNeedDump()) {
if (DumpManager::GetInstance().GetDumpProperties(kInferSessionId).IsSingleOpNeedDump()) {
GELOGI("Dump is open in single op, start to set dump info");
std::vector<uint64_t> input_addrs;
std::vector<uint64_t> output_adds;
@@ -68,7 +69,7 @@ Status OpTask::OpenDump(rtStream_t stream) {
uint64_t output_addr = arg_base[input_size + j];
output_adds.emplace_back(output_addr);
}
dump_op_.SetDumpInfo(DumpManager::GetInstance().GetDumpProperties(), op_desc_, input_addrs, output_adds, stream);
dump_op_.SetDumpInfo(DumpManager::GetInstance().GetDumpProperties(kInferSessionId), op_desc_, input_addrs, output_adds, stream);
auto status = dump_op_.LaunchDumpOp();
if (status != SUCCESS) {
GELOGE(status, "Launch dump op failed in single op");
@@ -194,11 +195,6 @@ Status TbeOpTask::LaunchKernel(rtStream_t stream) {
return RT_ERROR_TO_GE_STATUS(ret);
}
GELOGI("[TASK_INFO] %s", this->stub_name_.c_str());
auto status = OpenDump(stream);
if (status != SUCCESS) {
GELOGE(status, "Open dump failed in the tbe single op %s", this->stub_name_.c_str());
return status;
}

return SUCCESS;
}
@@ -491,6 +487,10 @@ Status AiCpuBaseTask::UpdateOutputShape(vector<GeTensorDesc> &output_desc) {
aicpu_ext_handle_->GetOutputShapeAndType(i, shape, data_type);
GE_CHK_STATUS_RET(UpdateShapeToOutputDesc(shape, output_desc[i]),
"AiCpuCCTask Update [%zu]th output shape failed.", i);
if (DumpManager::GetInstance().GetDumpProperties(kInferSessionId).IsSingleOpNeedDump()) {
GE_CHK_STATUS_RET(op_desc_->UpdateOutputDesc(i, output_desc[i]),
"AiCpuCCTask Update [%zu]th output desc failed.", i);
}
}
GELOGD("Update DEPEND_SHAPE_RANGE AiCpuBaseTask outputshape finished.");
return SUCCESS;
@@ -601,12 +601,6 @@ Status AiCpuTask::LaunchKernel(rtStream_t stream) {
}
GELOGI("[TASK_INFO] %lu/%s", kernel_id_, op_type_.c_str());

auto status = OpenDump(stream);
if (status != SUCCESS) {
GELOGE(status, "Open dump failed in aicpu single op %s", this->op_type_.c_str());
return status;
}

GELOGD("Done launch kernel successfully. task = %s", this->op_type_.c_str());
return SUCCESS;
}
@@ -700,6 +694,10 @@ Status AiCpuTask::UpdateShapeByHbmBuffer(vector<GeTensorDesc> &output_desc) {

GE_CHK_STATUS_RET(UpdateShapeToOutputDesc(GeShape(shape_dims), output_desc[i]),
"AiCpuTask update [%zu]th output shape failed.", i);
if (DumpManager::GetInstance().GetDumpProperties(kInferSessionId).IsSingleOpNeedDump()) {
GE_CHK_STATUS_RET(op_desc_->UpdateOutputDesc(i, output_desc[i]),
"AiCpuTask update [%zu]th output desc failed.", i);
}
}
return SUCCESS;
}
@@ -876,12 +874,6 @@ Status AiCpuCCTask::LaunchKernel(rtStream_t stream) {
}
GELOGI("[TASK_INFO] %lu/%s", kernel_id_, op_type_.c_str());
GELOGD("Invoke rtCpuKernelLaunch succeeded");
auto status = OpenDump(stream);
if (status != SUCCESS) {
GELOGE(status, "Open dump failed in the aicpucc single op %s", this->kernel_name_.c_str());
return status;
}

return SUCCESS;
}



+ 2
- 0
tests/ut/ge/CMakeLists.txt View File

@@ -354,6 +354,7 @@ set(COMMON_FORMAT_SRC_FILES
"${GE_CODE_DIR}/ge/common/formats/format_transfers/format_transfer_fracz_hwcn.cc"
"${GE_CODE_DIR}/ge/common/formats/utils/formats_trans_utils.cc"
"${GE_CODE_DIR}/ge/graph/manager/util/hcom_util.cc"
"${GE_CODE_DIR}/ge/common/dump/dump_manager.cc"
)

set(GRAPH_OPTIMIZE_COMMON_SRC_FILES
@@ -730,6 +731,7 @@ set(MULTI_PARTS_TEST_FILES
"graph_ir/ge_operator_factory_unittest.cc"
"graph/transop_util_unittest.cc"
"common/datatype_transfer_unittest.cc"
"common/dump_manager_unittest.cc"
"common/format_transfer_unittest.cc"
"common/format_transfer_transpose_unittest.cc"
"common/format_transfer_nchw_5d_unittest.cc"


+ 76
- 0
tests/ut/ge/common/dump_manager_unittest.cc View File

@@ -0,0 +1,76 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#include <gtest/gtest.h>

#include "common/dump/dump_manager.h"
#include "common/debug/log.h"
#include "common/ge_inner_error_codes.h"

namespace ge {
class UTEST_dump_manager : public testing::Test {
protected:
void SetUp() {}
void TearDown() {}
};
TEST_F(UTEST_dump_manager, is_dump_open_success) {
DumpConfig dump_config;
dump_config.dump_path = "/test";
dump_config.dump_mode = "all";
dump_config.dump_status = "on";
dump_config.dump_op_switch = "on";
auto ret = DumpManager::GetInstance().SetDumpConf(dump_config);
auto dump = DumpManager::GetInstance().GetDumpProperties(0);
bool result = dump.IsDumpOpen();
dump.ClearDumpInfo();
EXPECT_EQ(result, true);
}

TEST_F(UTEST_dump_manager, is_dump_op_success) {
DumpConfig dump_config;
dump_config.dump_path = "/test";
dump_config.dump_mode = "all";
dump_config.dump_status = "off";
auto ret = DumpManager::GetInstance().SetDumpConf(dump_config);
EXPECT_EQ(ret, ge::SUCCESS);
}

TEST_F(UTEST_dump_manager, is_dump_single_op_close_success) {
DumpConfig dump_config;
dump_config.dump_path = "/test";
dump_config.dump_mode = "all";
dump_config.dump_status = "on";
dump_config.dump_op_switch = "off";
auto ret = DumpManager::GetInstance().SetDumpConf(dump_config);
EXPECT_EQ(ret, ge::PARAM_INVALID);
}

TEST_F(UTEST_dump_manager, dump_status_empty) {
DumpConfig dump_config;
dump_config.dump_path = "/test";
dump_config.dump_mode = "all";
dump_config.dump_op_switch = "off";
auto ret = DumpManager::GetInstance().SetDumpConf(dump_config);
EXPECT_EQ(ret, ge::SUCCESS);
}

TEST_F(UTEST_dump_manager, add_dump_properties_success) {
DumpProperties dump_properties;
DumpManager::GetInstance().AddDumpProperties(0, dump_properties);
auto dump = DumpManager::GetInstance().GetDumpProperties(0);
DumpManager::GetInstance().RemoveDumpProperties(0);
}
} // namespace ge

Loading…
Cancel
Save