Browse Source

!914 bugfix for l1 data dump

From: @ni100die
Reviewed-by: @xchu42,@wqtshg
Signed-off-by: @wqtshg
tags/v1.2.0
mindspore-ci-bot Gitee 3 years ago
parent
commit
b82eaf3caa
2 changed files with 28 additions and 14 deletions
  1. +26
    -14
      ge/graph/load/new_model_manager/davinci_model.cc
  2. +2
    -0
      ge/graph/load/new_model_manager/davinci_model.h

+ 26
- 14
ge/graph/load/new_model_manager/davinci_model.cc View File

@@ -520,6 +520,8 @@ Status DavinciModel::DoTaskSink() {

GE_CHK_STATUS_RET(InitEntryTask(), "InitEntryTask failed.");

GE_CHK_STATUS_RET(InitL1DataDumperArgs(), "InitL1DataDumperArgs failed.");

GE_CHK_STATUS_RET(DistributeTask(), "Distribute failed.");

GE_CHK_RT_RET(rtModelLoadComplete(rt_model_handle_));
@@ -716,19 +718,6 @@ Status DavinciModel::Init(void *dev_ptr, size_t mem_size, void *weight_ptr, size
GE_CHK_STATUS_RET(DoTaskSink(), "Task sink failed");
GE_TIMESTAMP_END(DoTaskSink, "GraphLoader::DoTaskSink");

auto all_dump_model = GetDumpProperties().GetAllDumpModel();
bool findByOmName = all_dump_model.find(om_name_) != all_dump_model.end();
bool findByModelName = all_dump_model.find(name_) != all_dump_model.end();
bool dump_l1fusion_op = (all_dump_model.find(ge::DUMP_ALL_MODEL) != all_dump_model.end()) ||
findByOmName || findByModelName;
if (dump_l1fusion_op) {
// malloc 2M for dump l1fusion op
GE_CHK_RT_RET(rtMalloc(&l1_fusion_addr_, kDumpL1FusionOpMByteSize, RT_MEMORY_DDR));

// send l1fusion dump addr to rts
GE_CHK_RT_RET(rtDumpAddrSet(rt_model_handle_, l1_fusion_addr_, kDumpL1FusionOpMByteSize, kDumpFlagOfL1Fusion));
}

/// In zero copy model, if a aicpu operator is connected to the first or last layer, before model execution,
/// the aicpu opertor needs to destroy history record, and update operator memory address.
/// The model with specified aicpu operators is only marked here, and destruction is in ModelManager::ExecuteModel().
@@ -3951,7 +3940,6 @@ void DavinciModel::SetDataDumperArgs(const ComputeGraphPtr &graph, const map<str
data_dumper_.SetOmName(om_name_);
data_dumper_.SetComputeGraph(graph);
data_dumper_.SetRefInfo(saved_task_addrs_);
data_dumper_.SetL1FusionAddr(l1_fusion_addr_);

int32_t device_id = 0;
rtError_t rt_ret = rtGetDevice(&device_id);
@@ -4161,4 +4149,28 @@ int64_t DavinciModel::GetFixedAddrsSize(string tensor_name) {
}
}

Status DavinciModel::InitL1DataDumperArgs() {
auto all_dump_model = GetDumpProperties().GetAllDumpModel();
bool find_by_om_name = all_dump_model.find(om_name_) != all_dump_model.end();
bool find_by_model_name = all_dump_model.find(name_) != all_dump_model.end();
bool dump_l1fusion_op =
(all_dump_model.find(ge::DUMP_ALL_MODEL) != all_dump_model.end()) || find_by_om_name || find_by_model_name;
if (dump_l1fusion_op) {
// malloc 2M for dump l1fusion op
GE_CHK_RT_RET(rtMalloc(&l1_fusion_addr_, kDumpL1FusionOpMByteSize, RT_MEMORY_DDR));

// send l1fusion dump addr to rts
if (rtDumpAddrSet(rt_model_handle_, l1_fusion_addr_, kDumpL1FusionOpMByteSize, kDumpFlagOfL1Fusion) !=
RT_ERROR_NONE) {
// l1_fusion_addr_ will be free when DavinciModel destruct
GELOGE(FAILED, "Call rtDumpAddrSet failed");
return FAILED;
}

// set addr for l1 data dump
data_dumper_.SetL1FusionAddr(l1_fusion_addr_);
}
return SUCCESS;
}

} // namespace ge

+ 2
- 0
ge/graph/load/new_model_manager/davinci_model.h View File

@@ -837,6 +837,8 @@ class DavinciModel {

void SetDataDumperArgs(const ComputeGraphPtr &graph, const map<string, OpDescPtr> &variable_by_name);

Status InitL1DataDumperArgs();

Status InitModelProfile();
Status SinkModelProfile();



Loading…
Cancel
Save