Merge branch 'master' of gitee.com:tangqunzhang/graphengine

ge/common/dump/dump_manager.cc

@@ -23,6 +23,7 @@ const char *const kDumpOFF = "OFF";
 const char *const kDumpoff = "off";
 const char *const kDumpOn = "on";
 const uint64_t kInferSessionId = 0;
+const uint32_t kAllOverflow = 3;
 }  // namespace
 namespace ge {
 FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY DumpManager &DumpManager::GetInstance() {
@@ -30,78 +31,103 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY DumpManager &DumpManager::GetIn
   return instance;
 }
 
-FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status DumpManager::SetDumpConf(const DumpConfig &dump_config) {
-  DumpProperties dump_properties;
-  std::string dump_status;
-  std::string dump_path;
-  std::string dump_mode;
-  std::string dump_op_switch;
-
-  if (dump_config.dump_status.empty()) {
+bool DumpManager::NeedDoDump(const DumpConfig &dump_config, DumpProperties &dump_properties) {
+  if (dump_config.dump_status.empty() && dump_config.dump_debug.empty()) {
     dump_properties_map_.emplace(kInferSessionId, dump_properties);
     GELOGI("Dump does not open");
-    return SUCCESS;
+    return false;
   }
-
-  dump_status = dump_config.dump_status;
-  GELOGI("Dump status is %s", dump_status.c_str());
-  if (dump_config.dump_status == kDumpoff || dump_config.dump_status == kDumpOFF) {
+  GELOGI("Dump status is %s, dump debug is %s.", dump_config.dump_status.c_str(), dump_config.dump_debug.c_str());
+  if ((dump_config.dump_status == kDumpoff || dump_config.dump_status == kDumpOFF) &&
+       dump_config.dump_debug == kDumpoff) {
     dump_properties.ClearDumpPropertyValue();
     dump_properties_map_.emplace(kInferSessionId, dump_properties);
-    return SUCCESS;
+    return false;
+  }
+  if (dump_config.dump_status == kDumpOn && dump_config.dump_debug == kDumpOn) {
+    GELOGW("Not support coexistence of dump debug and dump status.");
+    return false;
   }
-  dump_properties.SetDumpStatus(dump_status);
+  return true;
+}
 
-  dump_op_switch = dump_config.dump_op_switch;
-  dump_properties.SetDumpOpSwitch(dump_op_switch);
-  if (dump_op_switch == kDumpoff && dump_config.dump_list.empty()) {
-    dump_properties_map_.emplace(kInferSessionId, dump_properties);
-    GELOGE(PARAM_INVALID, "[Check][DumpList]Invalid, dump_op_switch is %s",
-           dump_op_switch.c_str());
-    REPORT_INNER_ERROR("E19999", "Dump list check invalid, dump_op_switch is %s",
-                       dump_op_switch.c_str());
-    return PARAM_INVALID;
+void DumpManager::SetDumpDebugConf(const DumpConfig &dump_config, DumpProperties &dump_properties) {
+  if (dump_config.dump_debug == kDumpOn) {
+    GELOGI("Only do overflow detection, dump debug is %s.", dump_config.dump_debug.c_str());
+    dump_properties.InitInferOpDebug();
+    dump_properties.SetOpDebugMode(kAllOverflow);
   }
+}
 
-  if (!dump_config.dump_list.empty()) {
-    for (auto model_dump : dump_config.dump_list) {
-      std::string model_name = model_dump.model_name;
-      GELOGI("Dump model is %s", model_name.c_str());
-      std::set<std::string> dump_layers;
-      for (auto layer : model_dump.layers) {
-        GELOGI("Dump layer is %s in model", layer.c_str());
-        dump_layers.insert(layer);
-      }
-      dump_properties.AddPropertyValue(model_name, dump_layers);
+void DumpManager::SetDumpList(const DumpConfig &dump_config, DumpProperties &dump_properties) {
+  for (const auto &model_dump : dump_config.dump_list) {
+    std::string model_name = model_dump.model_name;
+    GELOGI("Dump model is %s", model_name.c_str());
+    std::set<std::string> dump_layers;
+    for (const auto &layer : model_dump.layers) {
+      GELOGI("Dump layer is %s in model", layer.c_str());
+      dump_layers.insert(layer);
+    }
+    dump_properties.AddPropertyValue(model_name, dump_layers);
+  }
+}
+
+Status DumpManager::SetNormalDumpConf(const DumpConfig &dump_config, DumpProperties &dump_properties) {
+  if (dump_config.dump_status == kDumpOn) {
+    GELOGI("Only do normal dump process, dump status is %s.", dump_config.dump_status.c_str());
+    dump_properties.SetDumpStatus(dump_config.dump_status);
+    std::string dump_op_switch = dump_config.dump_op_switch;
+    dump_properties.SetDumpOpSwitch(dump_op_switch);
+    if (dump_op_switch == kDumpoff && dump_config.dump_list.empty()) {
+      dump_properties_map_.emplace(kInferSessionId, dump_properties);
+      GELOGE(PARAM_INVALID, "[Check][DumpList]Invalid, dump_op_switch is %s", dump_op_switch.c_str());
+      REPORT_INNER_ERROR("E19999", "Dump list check invalid, dump_op_switch is %s", dump_op_switch.c_str());
+      return PARAM_INVALID;
     }
-    if (dump_op_switch == kDumpOn) {
-      GELOGI("Start to dump model and single op,dump op switch is %s", dump_op_switch.c_str());
+
+    if (!dump_config.dump_list.empty()) {
+      if (dump_op_switch == kDumpOn) {
+        GELOGI("Start to dump model and single op, dump op switch is %s", dump_op_switch.c_str());
+      } else {
+        GELOGI("Only dump model, dump op switch is %s", dump_op_switch.c_str());
+      }
+      SetDumpList(dump_config, dump_properties);
     } else {
-      GELOGI("Only dump model,dump op switch is %s", dump_op_switch.c_str());
+      GELOGI("Only dump single op, dump op switch is %s", dump_op_switch.c_str());
     }
-  } else {
-    GELOGI("Only dump single op,dump op switch is %s", dump_op_switch.c_str());
+    GELOGI("Dump mode is %s", dump_config.dump_mode.c_str());
+    dump_properties.SetDumpMode(dump_config.dump_mode);
   }
+  return SUCCESS;
+}
 
-  dump_path = dump_config.dump_path;
+Status DumpManager::SetDumpPath(const DumpConfig &dump_config, DumpProperties &dump_properties) {
+  std::string dump_path = dump_config.dump_path;
   if (dump_path.empty()) {
     GELOGE(PARAM_INVALID, "[Check][DumpPath]It is empty");
     REPORT_INNER_ERROR("E19999", "Dump path check is empty");
     return PARAM_INVALID;
   }
-
   if (dump_path[dump_path.size() - 1] != '/') {
     dump_path = dump_path + "/";
   }
   dump_path = dump_path + CurrentTimeInStr() + "/";
   GELOGI("Dump path is %s", dump_path.c_str());
   dump_properties.SetDumpPath(dump_path);
+  return SUCCESS;
+}
 
-  dump_mode = dump_config.dump_mode;
-  GELOGI("Dump mode is %s", dump_mode.c_str());
-  dump_properties.SetDumpMode(dump_mode);
+FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status DumpManager::SetDumpConf(const DumpConfig &dump_config) {
+  DumpProperties dump_properties;
+  if (!NeedDoDump(dump_config, dump_properties)) {
+    GELOGD("No need do dump process.");
+    return SUCCESS;
+  }
+  SetDumpDebugConf(dump_config, dump_properties);
+  GE_CHK_STATUS_RET(SetNormalDumpConf(dump_config, dump_properties), "[Init][DumpConf] failed when dump status is on.");
+  GE_CHK_STATUS_RET(SetDumpPath(dump_config, dump_properties), "[Init][DumpPath] failed.");
   dump_properties_map_[kInferSessionId] = dump_properties;
-
+  
   return SUCCESS;
 }
 

ge/common/dump/dump_manager.h

@@ -34,6 +34,11 @@ class DumpManager {
   void RemoveDumpProperties(uint64_t session_id);
 
  private:
+  bool NeedDoDump(const DumpConfig &dump_config, DumpProperties &dump_properties);
+  void SetDumpDebugConf(const DumpConfig &dump_config, DumpProperties &dump_properties);
+  Status SetDumpPath(const DumpConfig &dump_config, DumpProperties &dump_properties);
+  Status SetNormalDumpConf(const DumpConfig &dump_config, DumpProperties &dump_properties);
+  void SetDumpList(const DumpConfig &dump_config, DumpProperties &dump_properties);
   std::mutex mutex_;
   std::map<uint64_t, DumpProperties> dump_properties_map_;
 };

ge/common/dump/dump_properties.cc

@@ -53,7 +53,8 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void DumpProperties::InitByOpti
   dump_path_.clear();
   dump_step_.clear();
   dump_mode_.clear();
-  is_op_debug_ = false;
+  is_train_op_debug_ = false;
+  is_infer_op_debug_ = false;
   op_debug_mode_ = 0;
 
   std::string enable_dump;
@@ -124,7 +125,8 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void DumpProperties::ClearDumpI
   dump_mode_.clear();
   dump_op_switch_.clear();
   dump_status_.clear();
-  is_op_debug_ = false;
+  is_train_op_debug_ = false;
+  is_infer_op_debug_ = false;
   op_debug_mode_ = 0;
 }
 
@@ -203,6 +205,14 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY const std::string &DumpProperti
   return dump_status_;
 }
 
+FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void DumpProperties::InitInferOpDebug() {
+  is_infer_op_debug_ = true;
+}
+
+FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void DumpProperties::SetOpDebugMode(const uint32_t &op_debug_mode) {
+  op_debug_mode_ = op_debug_mode;
+}
+
 FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void DumpProperties::SetDumpOpSwitch(
   const std::string &dump_op_switch) {
   dump_op_switch_ = dump_op_switch;
@@ -237,7 +247,8 @@ void DumpProperties::CopyFrom(const DumpProperties &other) {
     dump_op_switch_ = other.dump_op_switch_;
 
     model_dump_properties_map_ = other.model_dump_properties_map_;
-    is_op_debug_ = other.is_op_debug_;
+    is_train_op_debug_ = other.is_train_op_debug_;
+    is_infer_op_debug_ = other.is_infer_op_debug_;
     op_debug_mode_ = other.op_debug_mode_;
   }
 }
@@ -254,15 +265,15 @@ void DumpProperties::SetDumpDebugOptions() {
 
     if (dump_debug_mode == OP_DEBUG_AICORE) {
       GELOGD("ge.exec.dumpDebugMode=aicore_overflow, op debug is open.");
-      is_op_debug_ = true;
+      is_train_op_debug_ = true;
       op_debug_mode_ = kAicoreOverflow;
     } else if (dump_debug_mode == OP_DEBUG_ATOMIC) {
       GELOGD("ge.exec.dumpDebugMode=atomic_overflow, op debug is open.");
-      is_op_debug_ = true;
+      is_train_op_debug_ = true;
       op_debug_mode_ = kAtomicOverflow;
     } else if (dump_debug_mode == OP_DEBUG_ALL) {
       GELOGD("ge.exec.dumpDebugMode=all, op debug is open.");
-      is_op_debug_ = true;
+      is_train_op_debug_ = true;
       op_debug_mode_ = kAllOverflow;
     } else {
       GELOGW("ge.exec.dumpDebugMode is invalid.");

ge/common/dump/dump_properties.h

@@ -65,16 +65,26 @@ class DumpProperties {
 
   const std::string &GetDumpStatus() const;
 
+  void InitInferOpDebug();
+
+  bool IsInferOpDebug() const {
+    return is_infer_op_debug_;
+  }
+
   void SetDumpOpSwitch(const std::string &dump_op_switch);
 
   const std::string &GetDumpOpSwitch() const;
 
-  bool IsOpDebugOpen() const { return is_op_debug_; }
+  bool IsOpDebugOpen() const {
+    return is_train_op_debug_ || is_infer_op_debug_;
+  }
 
   bool IsDumpOpen() const;
 
   bool IsSingleOpNeedDump() const;
 
+  void SetOpDebugMode(const uint32_t &op_debug_mode);
+
   uint32_t GetOpDebugMode() const { return op_debug_mode_; }
 
   const std::string &GetEnableDump() const {return enable_dump_;}
@@ -96,7 +106,8 @@ class DumpProperties {
   std::string dump_op_switch_;
   std::map<std::string, std::set<std::string>> model_dump_properties_map_;
 
-  bool is_op_debug_ = false;
+  bool is_train_op_debug_ = false;
+  bool is_infer_op_debug_ = false;
   uint32_t op_debug_mode_ = 0;
 };
 }

ge/graph/load/model_manager/data_dumper.cc

@@ -663,7 +663,7 @@ Status DataDumper::LoadDumpInfo() {
   SetOpDebugIdToAicpu(op_debug_task_id_, op_debug_stream_id_, op_debug_addr_, op_mapping_info);
 
   if (!op_list_.empty() || is_op_debug_ || is_end_graph_) {
-    auto ret = ExecuteLoadDumpInfo(op_mapping_info);
+    ret = ExecuteLoadDumpInfo(op_mapping_info);
     if (ret != SUCCESS) {
       GELOGE(ret, "Execute load dump info failed");
       return ret;

ge/graph/load/model_manager/davinci_model.cc

@@ -280,7 +280,7 @@ void DavinciModel::UnbindHcomStream() {
   if (!all_hccl_stream_list_.empty()) {
     for (size_t i = 0; i < all_hccl_stream_list_.size(); i++) {
       GE_LOGW_IF(rtModelUnbindStream(rt_model_handle_, all_hccl_stream_list_[i]) != RT_ERROR_NONE,
-                 "Unbind hccl stream from model failed! Index: %zu", i);
+                 "Unbind hccl stream from model failed, Index: %zu", i);
       GE_LOGW_IF(rtStreamDestroy(all_hccl_stream_list_[i]) != RT_ERROR_NONE, "Destroy hccl stream for rt_model failed")
     }
   }
@@ -551,7 +551,7 @@ Status DavinciModel::DoTaskSink() {
   }
 
   GE_CHK_RT_RET(rtGetAicpuDeploy(&deploy_type_));
-  GELOGI("Do task_sink. AiCpu deploy type is: %x.", deploy_type_);
+  GELOGI("Do task sink. AiCpu deploy type is: %x.", deploy_type_);
 
   GE_CHK_STATUS_RET(BindModelStream(), "Bind model stream failed.");
 
@@ -582,7 +582,7 @@ Status DavinciModel::SetTSDevice() {
   int64_t value = 0;
   bool ret = ge::AttrUtils::GetInt(ge_model_, ATTR_MODEL_CORE_TYPE, value);
   uint32_t core_type = ret ? static_cast<uint32_t>(value) : 0;
-  GELOGD("SetTSDevice: %u.", core_type);
+  GELOGD("Set TSDevice: %u.", core_type);
   rtError_t rt_ret = rtSetTSDevice(core_type);
   if (rt_ret != RT_ERROR_NONE) {
     REPORT_CALL_ERROR("E19999", "Call rtSetTSDevice failed, core_type:%u, model_id:%u",
@@ -969,7 +969,7 @@ Status DavinciModel::InitDataOp(const ComputeGraphPtr &graph, const NodePtr &nod
     return SUCCESS;
   }
 
-  GELOGI("Init Data node: %s.", op_desc->GetName().c_str());
+  GELOGI("Init data node: %s.", op_desc->GetName().c_str());
   auto data_index = data_op_index++;
   if (AttrUtils::GetInt(op_desc, ATTR_NAME_INDEX, data_index)) {
     GELOGD("Get new index %u, old %u", data_index, data_op_index - 1);
@@ -1027,7 +1027,7 @@ Status DavinciModel::GenInputOutputInfo(const map<uint32_t, OpDescPtr> &data_by_
   GELOGD("Data node size: %zu, NetOutput node size: %zu", data_by_index.size(), output_op_list.size());
   for (auto &item : data_by_index) {
     const auto output_addrs = ModelUtils::GetOutputDataAddrs(runtime_param_, item.second);
-    GELOGD("Data node: %s, output addr size: %zu", item.second->GetName().c_str(), output_addrs.size());
+    GELOGD("Data node is: %s, output addr size: %zu", item.second->GetName().c_str(), output_addrs.size());
     input_addrs_list_.emplace_back(output_addrs);
 
     GE_CHK_STATUS_RET(InitAippInfo(item.first, item.second), "Init AIPP Info failed");
@@ -1043,10 +1043,10 @@ Status DavinciModel::GenInputOutputInfo(const map<uint32_t, OpDescPtr> &data_by_
 
   vector<string> out_node_name;
   (void)AttrUtils::GetListStr(ge_model_, ATTR_MODEL_OUT_NODES_NAME, out_node_name);
-  GELOGD("Output node size: %zu, out nodes name: %zu", output_op_list.size(), out_node_name.size());
+  GELOGD("Output node size: %zu, out nodes name is: %zu", output_op_list.size(), out_node_name.size());
   for (const auto &op_desc : output_op_list) {
     const auto input_addrs = ModelUtils::GetInputDataAddrs(runtime_param_, op_desc);
-    GELOGD("NetOutput node: %s, input addr size: %zu", op_desc->GetName().c_str(), input_addrs.size());
+    GELOGD("NetOutput node is: %s, input addr size: %zu", op_desc->GetName().c_str(), input_addrs.size());
     output_addrs_list_.emplace_back(input_addrs);
 
     bool getnext_sink_dynamic = false;

ge/graph/load/model_manager/task_info/kernel_ex_task_info.cc

@@ -239,10 +239,6 @@ Status KernelExTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davin
                     return RT_ERROR_TO_GE_STATUS(rt_ret);)
 
     InitDumpTask(input_output_addr_, op_desc);
-    if (davinci_model_->GetOpDugReg()) {
-      GELOGI("Op debug is open in kernel ex task info");
-      dump_args_ = input_output_addr_;
-    }
   }
 
   uint64_t input_output_addr = static_cast<uint64_t>(reinterpret_cast<uintptr_t>(input_output_addr_));
@@ -275,10 +271,15 @@ Status KernelExTaskInfo::Init(const domi::TaskDef &task_def, DavinciModel *davin
 }
 
 void KernelExTaskInfo::InitDumpTask(void *addr, const OpDescPtr &op_desc) {
-  if (davinci_model_->OpNeedDump(op_desc->GetName())) {
+  if (davinci_model_->OpNeedDump(op_desc->GetName()) || davinci_model_->GetOpDugReg()) {
+    GELOGD("Op %s need dump in kernel ex task info", op_desc->GetName().c_str());
     dump_flag_ = RT_KERNEL_DUMPFLAG;
     dump_args_ = addr;
   }
+  if (davinci_model_->GetOpDugReg()) {
+    GELOGD("Op debug is open in kernel ex task info");
+    dump_args_ = addr;
+  }
 }
 
 Status KernelExTaskInfo::CalculateArgs(const domi::TaskDef &task_def, DavinciModel *davinci_model) {

ge/graph/load/model_manager/task_info/kernel_task_info.cc

@@ -731,9 +731,6 @@ Status KernelTaskInfo::InitTVMTask(uint16_t offset, const domi::KernelDef &kerne
   skt_dump_args_ = static_cast<char *>(args_) + offset;
   InitDumpTask(offset);
 
-  GE_CHK_BOOL_TRUE_EXEC_INFO(davinci_model_->GetOpDugReg(), dump_args_ = static_cast<char *>(args_) + offset,
-                             "Op debug is open in TVM task info");
-
   vector<void *> virtual_io_addrs;  // use virtual address for zero copy key.
   virtual_io_addrs.insert(virtual_io_addrs.end(), input_data_addrs.begin(), input_data_addrs.end());
   virtual_io_addrs.insert(virtual_io_addrs.end(), output_data_addrs.begin(), output_data_addrs.end());
@@ -1070,10 +1067,7 @@ Status KernelTaskInfo::InitAicpuTask(uint32_t op_index, const domi::KernelDef &k
     return RT_ERROR_TO_GE_STATUS(rt_ret);
   }
   InitDumpTask(sizeof(aicpu::AicpuParamHead));
-  if (davinci_model_->GetOpDugReg()) {
-    GELOGI("Op debug is open in aicpu task info");
-    dump_args_ = static_cast<char *>(args_) + sizeof(aicpu::AicpuParamHead);
-  }
+  
   if (kernel_type_ == ccKernelType::CUST_AI_CPU) {
     dump_flag_ |= RT_KERNEL_CUSTOM_AICPU;
   }
@@ -1085,6 +1079,7 @@ Status KernelTaskInfo::InitAicpuTask(uint32_t op_index, const domi::KernelDef &k
 
 void KernelTaskInfo::InitDumpTask(uint32_t offset) {
   if (davinci_model_->OpNeedDump(op_desc_->GetName())) {
+    GELOGD("Op %s need dump in task info", op_desc_->GetName().c_str());
     if (IsL1FusionOp(op_desc_)) {
       dump_flag_ = RT_FUSION_KERNEL_DUMPFLAG;
     } else {
@@ -1092,6 +1087,10 @@ void KernelTaskInfo::InitDumpTask(uint32_t offset) {
     }
     dump_args_ = static_cast<char *>(args_) + offset;
   }
+  if (davinci_model_->GetOpDugReg()) {
+    GELOGD("Op debug is open in kernel task info");
+    dump_args_ = static_cast<char *>(args_) + offset;
+  }
 }
 
 Status KernelTaskInfo::InitAicpuTaskExtInfo(const std::string &ext_info) {

ge/graph/manager/graph_manager.cc

@@ -492,9 +492,12 @@ Status GraphManager::AddGraph(const GraphId &graph_id, const Graph &graph,
   }
   // Do add graph
   SetAddGraphCondition(graph_id, kStartAdd);
+  if (CheckGraphAdded(graph_id, graph) != SUCCESS) {
+    GELOGE(FAILED, "AddGraph failed.");
+    return FAILED;
+  }
   auto compute_graph = GraphUtils::GetComputeGraph(graph);
   GE_CHECK_NOTNULL(compute_graph);
-  compute_graph->SetGraphID(graph_id);
   (void)AttrUtils::SetBool(*compute_graph, ATTR_NAME_GRAPH_HAS_BEEN_ADDED, true);
   SetSessionGraphId(compute_graph, graph_id);
 
@@ -552,6 +555,10 @@ Status GraphManager::CheckGraphAdded(const GraphId &graph_id, const Graph &graph
 Status GraphManager::AddGraphWithCopy(const GraphId &graph_id, const Graph &graph,
                                       const std::map<std::string, std::string> &options,
                                       const OmgContext &omg_context) {
+  if (HasGraphNode(graph_id)) {
+    GELOGE(GE_GRAPH_GRAPH_ALREADY_EXIST, "[GraphManager] graph exists, graph_id = %u", graph_id);
+    return GE_GRAPH_GRAPH_ALREADY_EXIST;
+  }
   if (CheckGraphAdded(graph_id, graph) != SUCCESS) {
     GELOGE(FAILED, "AddGraphWithCopy failed.");
     return FAILED;

ge/graph/manager/host_mem_manager.cc

@@ -45,12 +45,11 @@ Status SharedMemAllocator::Allocate(SharedMemInfo &mem_info) {
     return GE_GRAPH_MEMORY_ALLOC_FAILED;
   }
   mem_info.fd = output_para.fd;
-  mem_info.host_aligned_ptr = AlignedPtr::BuildFromAllocFunc([&output_para](std::unique_ptr<uint8_t[], deleter> &ptr) {
-                                                               ptr.reset(reinterpret_cast<uint8_t *>(output_para.ptr));
-                                                             },
-                                                             [](uint8_t *ptr) {
-                                                               ptr = nullptr;
-                                                             });
+  mem_info.host_aligned_ptr = AlignedPtr::BuildFromAllocFunc(
+    [&output_para](std::unique_ptr<uint8_t[], AlignedPtr::Deleter> &ptr) {
+      ptr.reset(reinterpret_cast<uint8_t *>(output_para.ptr));
+    },
+    [](uint8_t *ptr) { ptr = nullptr; });
   mem_info.device_address = reinterpret_cast<uint8_t *>(output_para.devPtr);
   return SUCCESS;
 }

ge/hybrid/executor/hybrid_model_async_executor.cc

@@ -544,9 +544,15 @@ Status HybridModelAsyncExecutor::DumpOpDebug() {
     data_dumper_.SetModelId(model_->GetModelId());
     data_dumper_.SetDeviceId(model_->GetDeviceId());
     void *global_step = nullptr;
-    TensorValue *varible_global_step = model_->GetVariable(NODE_NAME_GLOBAL_STEP);
-    if (varible_global_step != nullptr) {
-      global_step = const_cast<void *>(varible_global_step->GetData());
+
+    if (dump_properties.IsInferOpDebug()) {
+      GELOGD("Init global step when infer with op debug.");
+      global_step = executor_->GetContext()->global_step;
+    } else {
+      TensorValue *varible_global_step = model_->GetVariable(NODE_NAME_GLOBAL_STEP);
+      if (varible_global_step != nullptr) {
+        global_step = const_cast<void *>(varible_global_step->GetData());
+      }
     }
 
     void *loop_per_iter = nullptr;

inc/framework/common/ge_types.h

@@ -293,6 +293,7 @@ struct DumpConfig {
   std::string dump_mode;
   std::string dump_status;
   std::string dump_op_switch;
+  std::string dump_debug;
   std::vector<ModelDumpConfig> dump_list;
 };
 }  // namespace ge

tests/ut/ge/common/dump_manager_unittest.cc

@@ -67,6 +67,35 @@ TEST_F(UTEST_dump_manager, is_dump_single_op_close_success) {
    EXPECT_EQ(ret, ge::SUCCESS);
  }
 
+ // dump_debug and debug_status are on
+  TEST_F(UTEST_dump_manager, dump_op_debug_on) {
+    DumpConfig dump_config;
+    dump_config.dump_debug = "on";
+    dump_config.dump_status = "on";
+    auto ret = DumpManager::GetInstance().SetDumpConf(dump_config);
+    EXPECT_EQ(ret, ge::SUCCESS);
+  }
+
+  // just dump_status is on
+  TEST_F(UTEST_dump_manager, dump_status_without_dump_list) {
+    DumpConfig dump_config;
+    dump_config.dump_status = "on";
+    auto ret = DumpManager::GetInstance().SetDumpConf(dump_config);
+    EXPECT_EQ(ret, ge::PARAM_INVALID);
+  }
+
+  // dump_status is on with dump_list
+  TEST_F(UTEST_dump_manager, dump_status_with_dump_list) {
+    DumpConfig dump_config;
+    dump_config.dump_status = "on";
+    ModelDumpConfig dump_list;
+    dump_list.model_name = "test";
+    dump_list.layers.push_back("first");
+    dump_config.dump_list.push_back(dump_list);
+    auto ret = DumpManager::GetInstance().SetDumpConf(dump_config);
+    EXPECT_EQ(ret, ge::PARAM_INVALID);
+  }
+
  TEST_F(UTEST_dump_manager, add_dump_properties_success) {
    DumpProperties dump_properties;
    DumpManager::GetInstance().AddDumpProperties(0, dump_properties);

tests/ut/ge/graph/manager/graph_manager_unittest.cc

@@ -206,6 +206,37 @@ TEST_F(UtestGraphManagerTest, test_add_graph_3) {
   EXPECT_EQ(status2, ge::SUCCESS);
 }
 
+TEST_F(UtestGraphManagerTest, test_add_graph_4) {
+  GraphId graph_id = 1;
+  GraphManager graph_manager;
+  // create graph
+  Graph graph("test_graph");
+  CreateGraph(graph);
+  auto compute_graph = GraphUtils::GetComputeGraph(graph);
+  (void)AttrUtils::SetBool(*compute_graph, ATTR_NAME_GRAPH_HAS_BEEN_ADDED, true);
+
+  std::map<std::string, std::string> options;
+  OmgContext context;
+  Status status = graph_manager.AddGraph(graph_id, graph, options, context);
+  EXPECT_NE(status, ge::SUCCESS);
+}
+
+TEST_F(UtestGraphManagerTest, test_add_graph_with_copy_1) {
+  GraphId graph_id = 1;
+  GraphManager graph_manager;
+  
+  // create graph
+  Graph graph("test_graph");
+  CreateGraph(graph);
+  GraphNodePtr graph_node = MakeShared<GraphNode>(graph_id);
+  graph_manager.graph_map_.insert({1, graph_node});
+
+  std::map<std::string, std::string> options;
+  OmgContext context;
+  Status status = graph_manager.AddGraphWithCopy(graph_id, graph, options, context);
+  EXPECT_NE(status, ge::SUCCESS);
+}
+
 TEST_F(UtestGraphManagerTest, test_remove_graph_1) {
   GraphId graph_id = 1;
   GraphManager graph_manager;