link so for ge_runner

4 years ago · f9389e9966
--- a/ge/CMakeLists.txt
+++ b/ge/CMakeLists.txt
@@ -115,9 +115,9 @@ set(EXECUTOR_SRC_LIST
    "common/ge/op_tiling_manager.cc"
    "common/ge/plugin_manager.cc"
    "common/profiling/ge_profiling.cc"
    #"common/profiling/profiling_manager.cc"
    "common/profiling/profiling_manager.cc"
    "executor/ge_executor.cc"
    #"ge_local_engine/engine/host_cpu_engine.cc"
    "ge_local_engine/engine/host_cpu_engine.cc"
    "graph/execute/graph_execute.cc"
    "graph/execute/model_executor.cc"
    "graph/load/graph_loader.cc"
@@ -165,6 +165,7 @@ set(EXECUTOR_SRC_LIST
    "graph/manager/util/debug.cc"
    #"graph/manager/util/hcom_util.cc"              # Just for runner.
    "graph/passes/pass_utils.cc"
    "host_kernels/kernel_utils.cc"
    "host_kernels/rank_kernel.cc"
    "host_kernels/shape_kernel.cc"
    "host_kernels/shape_n_kernel.cc"
@@ -205,7 +206,7 @@ set(EXECUTOR_SRC_LIST
    "hybrid/node_executor/rts/rts_node_task.cc"
    "hybrid/node_executor/rts/rts_task_factory.cc"
    "hybrid/node_executor/task_context.cc"
    #"opskernel_manager/ops_kernel_builder_manager.cc"
    "opskernel_manager/ops_kernel_builder_manager.cc"
    "single_op/single_op.cc"
    "single_op/single_op_manager.cc"
    "single_op/single_op_model.cc"
@@ -224,9 +225,8 @@ set(COMPILER_SRC_LIST
    "common/dump/dump_op.cc"
    "common/ge/op_tiling_manager.cc"
    "common/ge/plugin_manager.cc"
    #"common/profiling/profiling_manager.cc"
    "engine_manager/dnnengine_manager.cc"
    #"ge_local_engine/engine/host_cpu_engine.cc"
    "ge_local_engine/engine/host_cpu_engine.cc"
    "ge_opt_info/ge_opt_info.cc"
    "generator/ge_generator.cc"
    "generator/generator_api.cc"
@@ -413,7 +413,7 @@ set(COMPILER_SRC_LIST
    "ir_build/attr_options/weight_compress_option.cc"
    "ir_build/ge_ir_build.cc"
    "ir_build/option_utils.cc"
    #"opskernel_manager/ops_kernel_builder_manager.cc"
    "opskernel_manager/ops_kernel_builder_manager.cc"
    "opskernel_manager/ops_kernel_manager.cc"
 )

--- a/ge/client/ge_api.cc
+++ b/ge/client/ge_api.cc
@@ -19,6 +19,7 @@
 #include <malloc.h>
 #include "framework/common/debug/log.h"
 #include "framework/common/debug/ge_log.h"
 #include "framework/executor/ge_executor.h"
 #include "common/ge/datatype_util.h"
 #include "proto/ge_api.pb.h"
 #include "graph/model_serialize.h"
@@ -161,6 +162,17 @@ Status GEInitializeImpl(const std::map<string, string> &options) {
    return ret;
  }

  ErrorManager::GetInstance().SetStage(error_message::kInitialize, error_message::kOther);
  GELOGI("GeExecutor initial.");
  GE_TIMESTAMP_START(GeExecutorInitialize);
  ret = GeExecutor::Initialize(options);
  if (ret != SUCCESS) {
    GELOGE(ret, "[Init][GeExecutor] GeExecutor initialize failed.");
    REPORT_CALL_ERROR("E19999", "GeExecutor initialize failed.");
    return ret;
  }
  GE_TIMESTAMP_END(GeExecutorInitialize, "GeExecutor::Initialize");

  // 7.check return status, return
  if (!g_ge_initialized) {
    // Initialize success, first time calling initialize
@@ -213,6 +225,7 @@ Status GEFinalize() {
  ErrorManager::GetInstance().GenWorkStreamIdDefault();
  GELOGT(TRACE_INIT, "GEFinalize start");

  (void)GeExecutor::FinalizeEx();
  GELOGI("SessionManager finalization.");
  if (g_session_manager != nullptr) {
    (void)g_session_manager->Finalize();  // always success.
--- a/ge/common/CMakeLists.txt
+++ b/ge/common/CMakeLists.txt
@@ -53,10 +53,6 @@ set(SRC_LIST
    "${GE_CODE_DIR}/ge/graph/manager/graph_var_manager.cc"
    "${GE_CODE_DIR}/ge/graph/manager/graph_manager_utils.cc"
    "${GE_CODE_DIR}/ge/graph/build/memory/var_mem_assign_util.cc"
    #### all in one test ####
    "${GE_CODE_DIR}/ge/common/profiling/profiling_manager.cc"
    "${GE_CODE_DIR}/ge/ge_local_engine/engine/host_cpu_engine.cc"
    "${GE_CODE_DIR}/ge/opskernel_manager/ops_kernel_builder_manager.cc" 
 )

 if (NOT ENABLE_D AND NOT ENABLE_ACL)
--- a/ge/common/profiling/profiling_manager.cc
+++ b/ge/common/profiling/profiling_manager.cc
@@ -18,13 +18,9 @@

 #include "framework/common/debug/ge_log.h"
 #include "framework/common/debug/log.h"
 #include "framework/common/string_util.h"
 #include "graph/ge_context.h"
 #include "graph/utils/type_utils.h"
 #include "external/graph/types.h"
 #include "runtime/base.h"
 #include "graph/load/model_manager/davinci_model.h"
 #include "mmpa/mmpa_api.h"

 namespace {
 const char *const kTrainingTrace = "training_trace";
@@ -82,14 +78,14 @@ ProfilingManager &ProfilingManager::Instance() {
  return profiling_manager;
 }

 ge::Status ProfilingManager::Init(const Options &options) {
 ge::Status ProfilingManager::Init(const string &mode, const string &options, const string &job_id) {
 #ifdef DAVINCI_SUPPORT_PROFILING
  vector<int32_t>().swap(device_id_);
  subscribe_count_ = 0;
  GELOGI("ProfilingManager::Init  job_id:%s", options.job_id.c_str());
  GELOGI("ProfilingManager::Init  job_id:%s", job_id.c_str());

  struct MsprofGeOptions prof_conf = {{ 0 }};
  Status ret = InitFromOptions(options, prof_conf);
  Status ret = InitFromOptions(mode, options, job_id, prof_conf);
  if (ret != SUCCESS) {
    GELOGE(ret, "[Init][Profiling]Failed, error_code %u", ret);
    REPORT_CALL_ERROR("E19999", "Init profiling failed, error_code %u", ret);
@@ -121,25 +117,52 @@ ge::Status ProfilingManager::Init(const Options &options) {
  return SUCCESS;
 }

 ge::Status ProfilingManager::InitFromOptions(const Options &options, MsprofGeOptions &prof_conf) {
 void ProfilingManager::Initialize(const map<string, string> &options) {
  GetContext().Init();

  auto it = options.find(OPTION_EXEC_JOB_ID);
  string job_id = (it != options.end()) ? it->second : "";

  it = options.find(OPTION_EXEC_SESSION_ID);
  string session_id = (it != options.end()) ? it->second : "";

  it = options.find(OPTION_EXEC_PROFILING_MODE);
  string profiling_mode = (it != options.end()) ? it->second : "";

  it = options.find(OPTION_EXEC_PROFILING_OPTIONS);
  string profiling_options = (it != options.end()) ? it->second : "";

  GELOGI("Init Profiling. session Id: %s", session_id.c_str());
  (void)Init(profiling_mode, profiling_options, job_id);
 }

 void ProfilingManager::Finalize() {
  if (ProfilingOn()) {
    StopProfiling();
    PluginUnInit();
  }
 }

 ge::Status ProfilingManager::InitFromOptions(const string &profiling_mode, const string &profiling_options,
                                             const string &job_id, MsprofGeOptions &prof_conf) {
 #ifdef DAVINCI_SUPPORT_PROFILING
  // enable profiling by env
  char env_profiling_mode[MMPA_MAX_PATH] = { 0x00 };
  is_execute_profiling_ = false;

  if (options.profiling_mode == "1" && !options.profiling_options.empty()) {
  if (profiling_mode == "1" && !profiling_options.empty()) {
    // enable profiling by ge option
    if (strncpy_s(prof_conf.options, MSPROF_OPTIONS_DEF_LEN_MAX, options.profiling_options.c_str(),
    if (strncpy_s(prof_conf.options, MSPROF_OPTIONS_DEF_LEN_MAX, profiling_options.c_str(),
                  MSPROF_OPTIONS_DEF_LEN_MAX - 1) != EOK) {
      GELOGE(INTERNAL_ERROR, "[copy][ProfilingOptions]Failed, options %s",
             options.profiling_options.c_str());
             profiling_options.c_str());
      REPORT_CALL_ERROR("E19999", "Copy profiling_options %s failed",
                        options.profiling_options.c_str());
                        profiling_options.c_str());
      return INTERNAL_ERROR;
    }
    is_execute_profiling_ = true;
    GELOGI("The profiling in options is %s, %s. origin option: %s", options.profiling_mode.c_str(), prof_conf.options,
           options.profiling_options.c_str());
    GELOGI("The profiling in options is %s, %s. origin option: %s", profiling_mode.c_str(), prof_conf.options,
           profiling_options.c_str());
  } else {
    (void)mmGetEnv("PROFILING_MODE", env_profiling_mode, MMPA_MAX_PATH);
    (void)mmGetEnv("PROFILING_OPTIONS", prof_conf.options, MSPROF_OPTIONS_DEF_LEN_MAX);
@@ -166,13 +189,13 @@ ge::Status ProfilingManager::InitFromOptions(const Options &options, MsprofGeOpt
    return ge::PARAM_INVALID;
  }

  if (strncpy_s(prof_conf.jobId, MSPROF_OPTIONS_DEF_LEN_MAX, options.job_id.c_str(), MSPROF_OPTIONS_DEF_LEN_MAX - 1) !=
  if (strncpy_s(prof_conf.jobId, MSPROF_OPTIONS_DEF_LEN_MAX, job_id.c_str(), MSPROF_OPTIONS_DEF_LEN_MAX - 1) !=
      EOK) {
    GELOGE(INTERNAL_ERROR, "[Copy][JobId]Failed, original job_id %s", options.job_id.c_str());
    REPORT_CALL_ERROR("E19999", "Copy job_id %s failed", options.job_id.c_str());
    GELOGE(INTERNAL_ERROR, "[Copy][JobId]Failed, original job_id %s", job_id.c_str());
    REPORT_CALL_ERROR("E19999", "Copy job_id %s failed", job_id.c_str());
    return INTERNAL_ERROR;
  }
  GELOGI("Job id: %s, original job id: %s.", prof_conf.jobId, options.job_id.c_str());
  GELOGI("Job id: %s, original job id: %s.", prof_conf.jobId, job_id.c_str());
 #endif
  return ge::SUCCESS;
 }
@@ -204,13 +227,14 @@ ge::Status ProfilingManager::ParseOptions(const std::string &options) {
      return ge::PARAM_INVALID;
    }
    if (prof_options.contains(kFpPoint)) {
      fp_point_ = prof_options[kFpPoint];
      domi::GetContext().forward_point = prof_options[kFpPoint];
    }
    if (prof_options.contains(kBpPoint)) {
      bp_point_ = prof_options[kBpPoint];
      domi::GetContext().backward_point = prof_options[kBpPoint];
    }
    if (!fp_point_.empty() && !bp_point_.empty()) {
      GELOGI("Training trace bp fp is set, bp_point:%s, fp_point:%s.", bp_point_.c_str(), fp_point_.c_str());
    if (!domi::GetContext().forward_point.empty() && !domi::GetContext().backward_point.empty()) {
      GELOGI("Training trace bp fp is set, bp_point:%s, fp_point:%s.",
             domi::GetContext().backward_point.c_str(), domi::GetContext().forward_point.c_str());
    }
    is_training_trace_ = true;
  } catch (...) {
@@ -593,6 +617,8 @@ Status ProfilingManager::ProfInit(uint64_t module) {
    is_training_trace_ = true;
  }
  GELOGI("Prof init success.");
  domi::GetContext().profiling_on = ProfilingOn();
  domi::GetContext().profiling_trace_on = ProfilingTrainingTraceOn();
 #endif
  return SUCCESS;
 }
@@ -1003,56 +1029,6 @@ void ProfilingManager::GetOpInputOutputInfo(const OpDescPtr &op, TaskDescInfo &t
  GetOpOutputInfo(op, task_desc_info);
 }

 void ProfilingManager::GetFpBpPoint(std::string &fp_point, std::string &bp_point) {
  // Env or options mode, fp_point_/bp_point_ have initiliazed on profiling init
  if (!fp_point_.empty() && !bp_point_.empty()) {
    fp_point = fp_point_;
    bp_point = bp_point_;
    GELOGI("Bp Fp have been initialized in env or options. bp_point: %s, fp_point: %s", bp_point.c_str(),
           fp_point.c_str());
    return;
  }
  // ProfApi mode and training trace is set
  // Parse options first
  char env_profiling_options[MSPROF_OPTIONS_DEF_LEN_MAX] = {0x00};
  bool is_profiling_valid = false;
  std::string profiling_options;
  if (ge::GetContext().GetOption(OPTION_EXEC_PROFILING_OPTIONS, profiling_options) == SUCCESS &&
      !profiling_options.empty()) {
    is_profiling_valid = true;
  } else {
    INT32 ret = mmGetEnv("PROFILING_OPTIONS", env_profiling_options, MSPROF_OPTIONS_DEF_LEN_MAX);
    if (ret != EN_OK) {
      GELOGI("PROFILING_OPTIONS env is not exist.");
      return;
    }
    GELOGI("Parse env PROFILING_OPTIONS:%s.", env_profiling_options);
    profiling_options = env_profiling_options;
    is_profiling_valid = true;
  }
  if (is_profiling_valid) {
    try {
      Json prof_options = Json::parse(profiling_options);
      if (prof_options.contains(kFpPoint)) {
        fp_point_ = prof_options[kFpPoint];
      }
      if (prof_options.contains(kBpPoint)) {
        bp_point_ = prof_options[kBpPoint];
      }
      fp_point = fp_point_;
      bp_point = bp_point_;
      if (!fp_point_.empty() && !bp_point_.empty()) {
        GELOGI("Training trace bp fp is set, bp_point:%s, fp_point:%s.", bp_point_.c_str(), fp_point_.c_str());
      }
    } catch (...) {
      GELOGW("Json prof options is invalid.");
      return;
    }
  }

  return;
 }

 FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::GetDeviceIdFromGraph(
    uint32_t graph_id, uint32_t &device_id) {
  auto iter = device_id_map_.find(graph_id);
--- a/ge/common/profiling/profiling_manager.h
+++ b/ge/common/profiling/profiling_manager.h
@@ -17,12 +17,12 @@
 #ifndef GE_COMMON_PROFILING_PROFILING_MANAGER_H_
 #define GE_COMMON_PROFILING_PROFILING_MANAGER_H_

 #include <nlohmann/json.hpp>
 #include <mutex>
 #include <map>
 #include <string>
 #include <vector>

 #include "nlohmann/json.hpp"
 #include "framework/common/ge_inner_error_codes.h"
 #include "framework/common/ge_types.h"
 #include "external/register/register_types.h"
@@ -78,14 +78,17 @@ class ProfilingManager {
  ProfilingManager();
  virtual ~ProfilingManager();
  static ProfilingManager &Instance();
  Status Init(const Options &options);

  void Initialize(const map<string, string> &options);
  void Finalize();

  Status ProfInit(uint64_t module);
  Status ProfFinalize();
  Status ProfStartProfiling(uint64_t module, const std::map<std::string, std::string> &config_para);
  Status ProfStopProfiling(uint64_t module, const std::map<std::string, std::string> &config_para);
  Status ProfModelSubscribe(uint64_t module, void *model);
  Status ProfModelUnsubscribe(void *model);
  void StopProfiling();

  bool ProfilingTrainingTraceOn() const { return is_training_trace_; }
  // report model load profiling data flag, data contain task desc info, step info, model load fusion op info
  bool ProfilingModelLoadOn() const { return is_load_profiling_; }
@@ -97,13 +100,12 @@ class ProfilingManager {
  void ProfilingTaskDescInfo(uint32_t model_id, const std::vector<TaskDescInfo> &task_desc_info,
                             const int32_t &device_id);
  void ProfilingOpInputOutInfo(const TaskDescInfo &task, Json &task_json);
  Status PluginInit();
  void PluginUnInit() const;

  Status CallMsprofReport(ReporterData &reporter_data) const;
  struct MsprofCallback &GetMsprofCallback() { return prof_cb_; }
  void SetMsprofCtrlCallback(MsprofCtrlCallback func) { prof_cb_.msprofCtrlCallback = func; }
  void SetMsprofReporterCallback(MsprofReporterCallback func) { prof_cb_.msprofReporterCallback = func; }
  void GetFpBpPoint(std::string &fp_point, std::string &bp_point);

  void GetOpInputOutputInfo(const OpDescPtr &op, TaskDescInfo &task_desc_info) const;
  void ReportData(const int32_t &device_id, const std::string &data, const std::string &tag_name);
  Status ProfileStepInfo(uint64_t index_id, uint64_t model_id, uint16_t tag_id, rtStream_t stream, int32_t device_id);
@@ -118,13 +120,19 @@ class ProfilingManager {
  Status GetModelIdFromGraph(uint32_t graph_id, uint32_t &model_id);

 private:
  Status InitFromOptions(const Options &options, MsprofGeOptions &prof_conf);
  Status Init(const string &mode, const string &options, const string &job_id);
  Status InitFromOptions(const string &mode, const string &options, const string &job_id, MsprofGeOptions &prof_conf);
  Status ParseOptions(const std::string &options);
  Status ProfParseParam(const std::map<std::string, std::string> &config_para, int32_t &device_num,
                        vector<int32_t> &device_list);
  Status ProfParseDeviceId(const std::map<std::string, std::string> &config_para,
                               vector<int32_t> &device_list);
  uint64_t GetProfilingModule();

  void StopProfiling();
  Status PluginInit();
  void PluginUnInit() const;

  void UpdateDeviceIdModuleMap(string prof_type, uint64_t module, const vector<int32_t> &device_list);
  void UpdateSubscribeDeviceModuleMap(std::string prof_type, uint32_t device_id, uint64_t module);
  void GetOpInputInfo(const OpDescPtr &op, TaskDescInfo &task_desc_info) const;
--- a/ge/executor/ge_executor.cc
+++ b/ge/executor/ge_executor.cc
@@ -28,6 +28,7 @@
 #include "graph/load/model_manager/model_manager.h"
 #include "graph/manager/graph_mem_manager.h"
 #include "graph/manager/graph_var_manager.h"
 #include "graph/manager/host_mem_manager.h"
 #include "single_op/single_op_manager.h"
 #include "graph/load/model_manager/davinci_model.h"
 #include "opskernel_manager/ops_kernel_builder_manager.h"
@@ -244,13 +245,13 @@ static void InitOpsProtoManager() {

 GeExecutor::GeExecutor() {}

 Status GeExecutor::Initialize() {
  GELOGI("Init GeExecutor begin.");
 Status GeExecutor::Initialize(const std::map<string, string> &options) {
  if (isInit_) {
    GELOGW("Already initialized, no need to be initialized again.");
    return ge::SUCCESS;
  }

  GELOGI("Init GeExecutor begin.");
  OpTilingManager::GetInstance().LoadSo();

  Status init_hostcpu_engine_status = HostCpuEngine::GetInstance().Initialize();
@@ -261,46 +262,61 @@ Status GeExecutor::Initialize() {

  InitOpsProtoManager();

  std::vector<rtMemType_t> mem_type(1, RT_MEMORY_HBM);
  mem_type.push_back(RT_MEMORY_P2P_DDR);
  auto ret = MemManager::Instance().Initialize(mem_type);
  if (ret != SUCCESS) {
    GELOGE(ret, "[Initialize][MemManager] failed.");
    return ret;
  GE_CHK_STATUS_RET(HostMemManager::Instance().Initialize());

  const std::vector<rtMemType_t> mem_type({1, RT_MEMORY_HBM, RT_MEMORY_P2P_DDR});
  Status status = MemManager::Instance().Initialize(mem_type);
  if (status != SUCCESS) {
    GELOGE(status, "[Init][MemManager] MemoryAllocatorManager initialize failed.");
    REPORT_CALL_ERROR("E19999", "MemManager initialize failed.");
    return status;
  }

  GE_CHK_STATUS_RET(OpsKernelBuilderManager::Instance().Initialize({}, false),
                    "[Initialize][OpsKernelBuilderManager] failed.");

  // Start profiling
  Options profiling_options;
  profiling_options.device_id = 0;
  // job id need to be set, the value is meaningless;
  profiling_options.job_id = "1";
  ProfilingManager::Instance().Init(profiling_options);
  const auto &model_manager = ModelManager::GetInstance();
  GE_CHECK_NOTNULL(model_manager);
  status = model_manager->EnableExceptionDump(options);
  if (status != SUCCESS) {
    GELOGW("[Init][ModelManager] Enable exception dump failed.");
  }

  ProfilingManager::Instance().Initialize(options);

  isInit_ = true;
  GELOGI("Init GeExecutor over.");
  return ge::SUCCESS;
  return SUCCESS;
 }

 Status GeExecutor::Finalize() {
  GELOGI("Uninit GeExecutor begin.");
 Status GeExecutor::FinalizeEx() {
  if (isInit_ == false) {
    GELOGW("GeExecutor has not been initialized.");
    return ge::SUCCESS;
  }

  (void) OpsKernelBuilderManager::Instance().Finalize();
  GELOGI("Uninit GeExecutor begin.");
  (void)OpsKernelBuilderManager::Instance().Finalize();

  // Stop profiling
  if (ProfilingManager::Instance().ProfilingOn()) {
    ProfilingManager::Instance().StopProfiling();
    ProfilingManager::Instance().PluginUnInit();
  }
  HostMemManager::Instance().Finalize();

  ProfilingManager::Instance().Finalize();

  GELOGI("Uninit GeExecutor over.");
  return ge::SUCCESS;
  return SUCCESS;
 }

 Status GeExecutor::Initialize() {
  // job id need to be set, the value is meaningless;
  const std::map<string, string> options({
      {OPTION_EXEC_JOB_ID, "1"}, {OPTION_EXEC_PROFILING_MODE, ""}, {OPTION_EXEC_PROFILING_OPTIONS, ""}
  });

  return GeExecutor::Initialize(options);
 }

 Status GeExecutor::Finalize() {
  return GeExecutor::FinalizeEx();
 }

 Status GeExecutor::SetDynamicBatchSize(uint32_t model_id, void *dynamic_input_addr, uint64_t length,
--- a/ge/graph/build/task_generator.cc
+++ b/ge/graph/build/task_generator.cc
@@ -17,7 +17,7 @@
 #include "graph/build/task_generator.h"
 #include <string>
 #include <utility>
 #include "common/profiling/profiling_manager.h"
 #include "nlohmann/json.hpp"
 #include "framework/common/types.h"
 #include "framework/common/util.h"
 #include "framework/common/debug/ge_log.h"
@@ -35,6 +35,7 @@
 #include "external/ge/ge_api_types.h"
 #include "opskernel_manager/ops_kernel_builder_manager.h"

 using Json = nlohmann::json;
 using domi::LogTimeStampDef;
 using domi::ModelTaskDef;
 using domi::TaskDef;
@@ -51,6 +52,9 @@ const char *const kIsOutputVar = "OUTPUT_IS_VAR";
 const char *const kProfilingMode = "PROFILING_MODE";
 const char *const kIteratorV2 = "IteratorV2";
 const char *const kKernelInfoNameHccl = "ops_kernel_info_hccl";
 const char *const kFpPoint = "fp_point";
 const char *const kBpPoint = "bp_point";

 const uint32_t kProfilingArStep = 2;
 const uint64_t kProfilingFpStartLogid = 1;
 const uint64_t kProfilingBpEndLogid = 2;
@@ -65,6 +69,8 @@ namespace ge {
 TaskGenerator::TaskGenerator(uint8_t *var_mem_base, uint64_t var_mem_size) {
  var_mem_base_ = var_mem_base;
  var_mem_size_ = var_mem_size;
  fp_point_ = domi::GetContext().forward_point;
  bp_point_ = domi::GetContext().backward_point;
 }
 TaskGenerator::~TaskGenerator() {}

@@ -941,12 +947,53 @@ Status TaskGenerator::FindBpOfEnv(const ComputeGraphPtr &graph, const std::strin
  return SUCCESS;
 }

 Status TaskGenerator::GetFpBpIndex(const ComputeGraphPtr &graph, ProfilingPoint &profiling_point,
                                   vector<uint32_t> &all_reduce_nodes, std::string &fp_point_str,
                                   std::string &bp_point_str) const {
 void TaskGenerator::GetFpBpPoint(std::string &fp_point, std::string &bp_point) {
  // Env or options mode, fp_point_/bp_point_ have initiliazed on profiling init
  if (!fp_point_.empty() && !bp_point_.empty()) {
    fp_point = fp_point_;
    bp_point = bp_point_;
    GELOGI("Bp[%s] Fp[%s] have been initialized in env or options.", bp_point.c_str(), fp_point.c_str());
    return;
  }

  // ProfApi mode and training trace is set
  // Parse options first
  std::string profiling_options;
  graphStatus get_status = GetContext().GetOption(OPTION_EXEC_PROFILING_OPTIONS, profiling_options);
  if (get_status != GRAPH_SUCCESS || profiling_options.empty()) {
    char env_profiling_options[MSPROF_OPTIONS_DEF_LEN_MAX] = {0x00};
    if (mmGetEnv("PROFILING_OPTIONS", env_profiling_options, MSPROF_OPTIONS_DEF_LEN_MAX) != EN_OK) {
      GELOGI("PROFILING_OPTIONS env is not exist.");
      return;
    }
    GELOGI("Parse env PROFILING_OPTIONS:%s.", env_profiling_options);
    profiling_options = env_profiling_options;
  }

  ProfilingManager::Instance().GetFpBpPoint(fp_point_str, bp_point_str);
  if (!profiling_options.empty()) {
    try {
      Json prof_options = Json::parse(profiling_options);
      if (prof_options.contains(kFpPoint)) {
        fp_point_ = prof_options[kFpPoint];
      }
      if (prof_options.contains(kBpPoint)) {
        bp_point_ = prof_options[kBpPoint];
      }
      fp_point = fp_point_;
      bp_point = bp_point_;
      if (!fp_point_.empty() && !bp_point_.empty()) {
        GELOGI("Training trace bp fp is set, bp_point:%s, fp_point:%s.", bp_point_.c_str(), fp_point_.c_str());
      }
    } catch (...) {
      GELOGW("Json prof options is invalid.");
      return;
    }
  }
 }

 Status TaskGenerator::GetFpBpIndex(const ComputeGraphPtr &graph, ProfilingPoint &profiling_point,
                                   vector<uint32_t> &all_reduce_nodes, const std::string &fp_point_str,
                                   const std::string &bp_point_str) const {
  Status ret = SUCCESS;
  if (fp_point_str.empty()) {
    ret = AutoFindFpOpIndex(graph, profiling_point);
@@ -973,11 +1020,11 @@ Status TaskGenerator::FindProfilingNodeIndex(const ComputeGraphPtr &graph, Profi
 }

 Status TaskGenerator::FindProfilingTaskIndex(const ComputeGraphPtr &graph, ProfilingPoint &profiling_point,
                                             vector<uint32_t> &all_reduce_nodes) const {
                                             vector<uint32_t> &all_reduce_nodes) {
  GE_CHECK_NOTNULL(graph);
  const char *profiling_mode = std::getenv(kProfilingMode);
  bool is_profiling = (profiling_mode != nullptr) || ProfilingManager::Instance().ProfilingOn() ||
                      ProfilingManager::Instance().ProfilingTrainingTraceOn();
  bool is_profiling = (profiling_mode != nullptr) || domi::GetContext().profiling_on ||
                      domi::GetContext().profiling_trace_on;
  if (!is_profiling) {
    GELOGD("Profiling is not open.");
    return SUCCESS;
@@ -992,6 +1039,7 @@ Status TaskGenerator::FindProfilingTaskIndex(const ComputeGraphPtr &graph, Profi
  GELOGI("Start get FP/BP index.");
  std::string fp_point_str;
  std::string bp_point_str;
  GetFpBpPoint(fp_point_str, bp_point_str);
  Status ret = GetFpBpIndex(graph, profiling_point, all_reduce_nodes, fp_point_str, bp_point_str);
  if (ret != SUCCESS) {
    GELOGW("Get FP_POINT BP_POINT failed.");
@@ -1071,8 +1119,8 @@ Status TaskGenerator::InsertProfilingTaskBefore(const OpDescPtr &op_desc, const
                                                vector<uint32_t> &all_reduce_nodes, uint32_t node_index,
                                                vector<domi::TaskDef> &task_def_list) {
  const char *profiling_mode = std::getenv(kProfilingMode);
  bool is_profiling = (profiling_mode != nullptr) || ProfilingManager::Instance().ProfilingOn() ||
                      ProfilingManager::Instance().ProfilingTrainingTraceOn();
  bool is_profiling = (profiling_mode != nullptr) || domi::GetContext().profiling_on ||
                      domi::GetContext().profiling_trace_on;
  bool is_insert_fp_profiling_task = false;
  (void)ge::AttrUtils::GetBool(op_desc, ATTR_NAME_INSERT_FP_PROFILILNG_TASK, is_insert_fp_profiling_task);
  bool is_insert_bp_profiling_task = false;
@@ -1167,8 +1215,8 @@ Status TaskGenerator::InsertProfilingTaskAfter(const OpDescPtr &op_desc, const P
                                               vector<domi::TaskDef> &task_def_list) {
  GE_CHECK_NOTNULL(op_desc);
  const char *profiling_mode = std::getenv(kProfilingMode);
  bool is_profiling = (profiling_mode != nullptr) || ProfilingManager::Instance().ProfilingOn() ||
                      ProfilingManager::Instance().ProfilingTrainingTraceOn();
  bool is_profiling = (profiling_mode != nullptr) || domi::GetContext().profiling_on ||
                      domi::GetContext().profiling_trace_on;
  bool is_insert_bp_profiling_task = false;
  (void)ge::AttrUtils::GetBool(op_desc, ATTR_NAME_INSERT_BP_PROFILILNG_TASK, is_insert_bp_profiling_task);
  bool is_insert_end_profiling_task = false;
--- a/ge/graph/build/task_generator.h
+++ b/ge/graph/build/task_generator.h
@@ -123,11 +123,12 @@ class TaskGenerator {
  Status FindBpOfEnv(const ComputeGraphPtr &graph, const std::string &bp_point_str, ProfilingPoint &profiling_point,
                     vector<uint32_t> &all_reduce_nodes) const;

  void GetFpBpPoint(std::string &fp_point, std::string &bp_point);
  Status GetFpBpIndex(const ComputeGraphPtr &graph, ProfilingPoint &profiling_point, vector<uint32_t> &all_reduce_nodes,
                      std::string& fp_point_str, std::string& bp_point_str) const;
                      const std::string &fp_point_str, const std::string &bp_point_str) const;

  Status FindProfilingTaskIndex(const ComputeGraphPtr &graph, ProfilingPoint &profiling_point,
                                std::vector<uint32_t> &all_reduce_nodes) const;
                                std::vector<uint32_t> &all_reduce_nodes);
  Status InsertProfilingTaskBefore(const OpDescPtr &op_desc, const ProfilingPoint &profiling_point,
                                   std::vector<uint32_t> &all_reduce_nodes, uint32_t node_index,
                                   std::vector<domi::TaskDef> &task_def_list);
@@ -164,6 +165,8 @@ class TaskGenerator {

  bool IsSubGraphOfDynamicGraph(const ComputeGraphPtr &graph) const;

  std::string fp_point_;
  std::string bp_point_;
  uint8_t *var_mem_base_ = nullptr;
  uint64_t var_mem_size_ = 0;
 };
--- a/ge/graph/execute/model_executor.cc
+++ b/ge/graph/execute/model_executor.cc
@@ -28,6 +28,7 @@
 #include "graph/load/model_manager/model_manager.h"
 #include "common/math/math_util.h"
 #include "common/formats/utils/formats_trans_utils.h"
 #include "common/profiling/profiling_manager.h"

 namespace {
 constexpr int32_t kBase = 10;
@@ -42,6 +43,12 @@ namespace ge {
 /// @return Status result of function
 ///
 Status ModelExecutor::Initialize(const map<string, string> &options, uint64_t session_id) {
  if (init_flag_) {
    GELOGW("ModelExecutor has already initialized.");
    return SUCCESS;
  }

  session_id_ = session_id;
  graph_run_listener_ = MakeShared<GraphModelListener>(sync_run_mutex_, condition_);
  if (graph_run_listener_ == nullptr) {
    REPORT_CALL_ERROR("E19999", "New GraphModelListener fail");
@@ -49,34 +56,15 @@ Status ModelExecutor::Initialize(const map<string, string> &options, uint64_t se
    return MEMALLOC_FAILED;
  }

  const auto model_manager = ModelManager::GetInstance();
  GE_CHECK_NOTNULL(model_manager);
  Status status = model_manager->EnableExceptionDump(options);
  if (status != SUCCESS) {
    return status;
  }

  GE_CHK_STATUS_RET(HostMemManager::Instance().Initialize());

  const std::vector<rtMemType_t> mem_type({RT_MEMORY_HBM, RT_MEMORY_P2P_DDR});
  status = MemManager::Instance().Initialize(mem_type);
  if (status != SUCCESS) {
    GELOGE(status, "[Init][MemManager] MemoryAllocatorManager initialize failed.");
    REPORT_CALL_ERROR("E19999", "MemManager initialize failed.");
    return status;
  }

  VarManager::Instance(session_id)->SetMemManager(&MemManager::Instance());
  size_t total_mem_size = 0;
  GE_CHK_STATUS_RET_NOLOG(GetTotalMemorySize(total_mem_size));
  status = VarManager::Instance(session_id)->SetMemoryMallocSize(options, total_mem_size);
  Status status = VarManager::Instance(session_id)->SetMemoryMallocSize(options, total_mem_size);
  if (status != SUCCESS) {
    GELOGE(status, "[Set][MemoryMallocSize] failed.");
    REPORT_CALL_ERROR("E19999", "VarManager SetMemoryMallocSize failed, InnerSession:%lu.", session_id_);
    return status;
  }

  session_id_ = session_id;
  train_graph_flag_ = ParseTrainGraphFlag();
  thread_run_flag_.store(true);
  run_thread_ = std::thread(&ModelExecutor::RunThread, this);
@@ -108,7 +96,6 @@ Status ModelExecutor::Finalize() {
  GELOGI("VarManager free var memory.");
  (void)VarManager::Instance(session_id_)->FreeVarMemory();
  MemManager::Instance().FreeSessionMemory(session_id_);
  HostMemManager::Instance().Finalize();

  ModelManager::GetInstance()->DestroyAicpuSession(session_id_);
  return SUCCESS;
@@ -415,6 +402,7 @@ Status ModelExecutor::ModelLoad(const GeRootModelPtr &ge_root_model, const Graph
  ge_root_model->SetModelId(model_id);
  graph_node->SetGeRootModel(ge_root_model);
  AddGraphNode(graph_node->GetGraphId(), graph_node);
  ProfilingManager::Instance().SetGraphIdToDeviceMap(graph_node->GetGraphId(), GetContext().DeviceId());
  return SUCCESS;
 }

--- a/ge/graph/load/model_manager/davinci_model.cc
+++ b/ge/graph/load/model_manager/davinci_model.cc
@@ -500,6 +500,8 @@ void DavinciModel::InitRuntimeParams() {

  ret = ge::AttrUtils::GetInt(ge_model_, ATTR_MODEL_ZERO_COPY_MEMORY_SIZE, value);
  runtime_param_.zero_copy_size = ret ? value : 0;

  VarManager::Instance(session_id_)->SetMemManager(&MemManager::Instance());
  GELOGI("InitRuntimeParams(), %s.", runtime_param_.ToString().c_str());
 }

--- a/ge/graph/manager/graph_manager.cc
+++ b/ge/graph/manager/graph_manager.cc
@@ -109,7 +109,6 @@
 #include "register/custom_pass_helper.h"
 #include "external/graph/types.h"
 #include "common/util/error_manager/error_manager.h"
 #include "common/profiling/profiling_manager.h"

 namespace {
 const char *const kSummary = "Summary";
@@ -462,9 +461,6 @@ Status GraphManager::AddGraph(const GraphId &graph_id, const Graph &graph,
                              const std::map<std::string, std::string> &options,
                              const OmgContext &omg_context) {
  IncreaseGraphCount(graph_id);
  auto device_id = GetContext().DeviceId();
  GELOGD("Device id is %u", device_id);
  ProfilingManager::Instance().SetGraphIdToDeviceMap(graph_id, device_id);
  // validation for adding graphs of same graph_id in multi-thread secenario
  // 1.previous thread owns same graph_id has finished the AddGraph procession
  if (GetAddGraphCondition(graph_id) == kDoneAdded) {
--- a/ge/graph/manager/graph_var_manager.cc
+++ b/ge/graph/manager/graph_var_manager.cc
@@ -623,6 +623,7 @@ rtMemType_t VarManager::GetVarMemType(const int64_t &offset) {
 void VarManager::SetMemManager(MemoryManager *mem_manager) {
  // Better use shared_ptr instead, reconsitution later.
  GELOGI("Set MemManager to VarManager.");
  std::lock_guard<std::recursive_mutex> lock(mutex_);
  mem_manager_ = mem_manager;
 }

--- a/ge/hybrid/hybrid_davinci_model_stub.cc
+++ b/ge/hybrid/hybrid_davinci_model_stub.cc
@@ -1,115 +0,0 @@
 /**
 * Copyright 2019-2020 Huawei Technologies Co., Ltd
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

 #include "hybrid/hybrid_davinci_model.h"

 namespace ge {
 namespace hybrid {
 HybridDavinciModel::~HybridDavinciModel() {}

 std::unique_ptr<HybridDavinciModel> HybridDavinciModel::Create(const GeRootModelPtr &ge_root_model) {
  return std::unique_ptr<HybridDavinciModel>(new (std::nothrow)HybridDavinciModel());
 }

 Status HybridDavinciModel::Init() {
  return UNSUPPORTED;
 }

 Status HybridDavinciModel::Execute(const std::vector<DataBuffer> &inputs,
                                   const std::vector<GeTensorDesc> &input_desc,
                                   std::vector<DataBuffer> &outputs,
                                   std::vector<GeTensorDesc> &output_desc,
                                   rtStream_t stream) {
  return UNSUPPORTED;
 }

 Status HybridDavinciModel::Execute(const vector<GeTensor> &inputs, vector<GeTensor> &outputs) {
  return UNSUPPORTED;
 }

 Status HybridDavinciModel::ModelRunStart() {
  return UNSUPPORTED;
 }

 Status HybridDavinciModel::ModelRunStop() {
  return UNSUPPORTED;
 }

 Status HybridDavinciModel::EnqueueData(const shared_ptr<InputDataWrapper> &data) {
  return UNSUPPORTED;
 }

 void HybridDavinciModel::SetListener(const shared_ptr<ModelListener> &listener) {
 }

 void HybridDavinciModel::SetModelId(uint32_t model_id) {
 }

 void HybridDavinciModel::SetDeviceId(uint32_t device_id) {
 }

 void HybridDavinciModel::SetOmName(const string &om_name) {
 }

 uint64_t HybridDavinciModel::GetSessionId() {
  return 0;
 }

 uint32_t HybridDavinciModel::GetDataInputerSize() {
  return 0;
 }

 uint32_t HybridDavinciModel::GetDeviceId() const {
  return 0;
 }

 Status HybridDavinciModel::GetDynamicBatchInfo(std::vector<std::vector<int64_t>> &batch_info, int32_t &dynamic_type) {
  return UNSUPPORTED;
 }

 void HybridDavinciModel::GetUserDesignateShapeOrder(std::vector<std::string> &user_input_shape_order) {
 }

 void HybridDavinciModel::GetModelAttr(std::vector<std::string> &dynamic_output_shape_info) {
 }

 Status HybridDavinciModel::GetInputOutputDescInfo(vector<InputOutputDescInfo> &input_desc,
                                                  vector<InputOutputDescInfo> &output_desc,
                                                  std::vector<uint32_t> &input_formats,
                                                  std::vector<uint32_t> &output_formats) {
  return UNSUPPORTED;
 }

 void HybridDavinciModel::SetModelDescVersion(bool is_new_model_desc) {
 }

 bool HybridDavinciModel::GetRunningFlag() const {
  return false;
 }

 Status HybridDavinciModel::SetRunAsyncListenerCallback(const RunAsyncCallback &callback) {
  return UNSUPPORTED;
 }

 bool HybridDavinciModel::GetOpDescInfo(uint32_t stream_id, uint32_t task_id, OpDescInfo &op_desc_info) const {
  return true;
 }
 Status HybridDavinciModel::GetOpAttr(const std::string &op_name, const std::string &attr_name,
                                     std::string &attr_value) const {
  return UNSUPPORTED;
 }
 }  // namespace hybrid
 }  // namespace ge
--- a/ge/hybrid/model/hybrid_model_builder.cc
+++ b/ge/hybrid/model/hybrid_model_builder.cc
@@ -1537,6 +1537,7 @@ Status HybridModelBuilder::InitRuntimeParams() {

  var_manager_ = VarManager::Instance(runtime_param_.session_id);
  GE_CHECK_NOTNULL(var_manager_);
  var_manager_->SetMemManager(&MemManager::Instance());
  return SUCCESS;
 }

--- a/ge/init/gelib.cc
+++ b/ge/init/gelib.cc
@@ -25,7 +25,6 @@

 #include "common/ge/ge_util.h"
 #include "common/ge/plugin_manager.h"
 #include "common/profiling/profiling_manager.h"
 #include "common/properties_manager.h"
 #include "framework/common/debug/ge_log.h"
 #include "framework/common/debug/log.h"
@@ -191,8 +190,6 @@ Status GELib::SystemInitialize(const map<string, string> &options) {

  InitOptions(options);

  // In train and infer, profiling is always needed.
  InitProfiling(this->options_);
  // 1.`is_train_mode_` means case: train
  // 2.`(!is_train_mode_) && (options_.device_id != kDefaultDeviceIdForInfer)` means case: online infer
  // these two case with logical device id
@@ -204,16 +201,6 @@ Status GELib::SystemInitialize(const map<string, string> &options) {
  return status;
 }

 void GELib::InitProfiling(Options &options) {
  GELOGI("Init Profiling. session Id: %ld, device id:%d ", options.session_id, options.device_id);
  std::lock_guard<std::mutex> lock(status_mutex_);
  GetContext().Init();
  // Profiling init
  if (ProfilingManager::Instance().Init(options) != SUCCESS) {
    GELOGW("Profiling init failed.");
  }
 }

 void GELib::SetDefaultPrecisionMode(map<string, string> &new_options) {
  auto iter = new_options.find(PRECISION_MODE);
  if (iter != new_options.end()) {
@@ -432,9 +419,6 @@ Status GELib::Finalize() {
  GELOGI("Analyzer finalization");
  Analyzer::GetInstance()->Finalize();

  // Shut down profiling
  ShutDownProfiling();

  if (is_train_mode_ || (options_.device_id != kDefaultDeviceIdForInfer)) {
    GELOGI("System ShutDown.");
    mid_state = SystemShutdownWithOptions(this->options_);
@@ -463,15 +447,6 @@ Status GELib::Finalize() {
  return SUCCESS;
 }

 void GELib::ShutDownProfiling() {
  std::lock_guard<std::mutex> lock(status_mutex_);

  if (ProfilingManager::Instance().ProfilingOn()) {
    ProfilingManager::Instance().StopProfiling();
    ProfilingManager::Instance().PluginUnInit();
  }
 }

 // Get Singleton Instance
 std::shared_ptr<GELib> GELib::GetInstance() { return instancePtr_; }

--- a/ge/init/gelib.h
+++ b/ge/init/gelib.h
@@ -61,12 +61,6 @@ class GE_FUNC_VISIBILITY GELib {
  // get Initial flag
  bool InitFlag() const { return init_flag_; }

  // get TrainMode flag
  bool IsTrainMode() { return is_train_mode_; }

  void InitProfiling(Options &options);
  void ShutDownProfiling();

  Status InitSystemWithoutOptions();
  Status InitSystemWithOptions(Options &options);
  Status SystemShutdownWithOptions(const Options &options);
--- a/ge/ir_build/ge_ir_build.cc
+++ b/ge/ir_build/ge_ir_build.cc
@@ -474,8 +474,8 @@ graphStatus Impl::CheckBuildModeAndBuildStep() {
    }
  } else {
    if (build_mode == BUILD_MODE_TUNING) {
      REPORT_INPUT_ERROR("E10001", std::vector<std::string>({"parameter", "value", "reason"}),
          std::vector<std::string>({BUILD_MODE, it->second, "tuning must specify build step. Please check!"}));
      //REPORT_INPUT_ERROR("E10001", std::vector<std::string>({"parameter", "value", "reason"}),
      //    std::vector<std::string>({BUILD_MODE, it->second, "tuning must specify build step. Please check!"}));
      GELOGE(GRAPH_PARAM_INVALID, "[Check][BuildMode] tuning must specify build step. Please check!");
      return GRAPH_PARAM_INVALID;
    }
--- a/ge/session/inner_session.cc
+++ b/ge/session/inner_session.cc
@@ -34,7 +34,6 @@
 #include "graph/utils/tensor_adapter.h"
 #include "runtime/mem.h"
 #include "ir_build/option_utils.h"
 #include "common/profiling/profiling_manager.h"

 namespace ge {
 namespace {
@@ -48,9 +47,9 @@ Status CheckReuseMemoryOption(const std::map<string, string> &options) {
    } else if (iter->second == "1") {
      GELOGD("%s=1, reuse memory is close", OPTION_EXEC_DISABLE_REUSED_MEMORY);
    } else {
      GELOGE(PARAM_INVALID, "[CheckReuse][MemoryOption]option %s=%s is invalid", 
      GELOGE(PARAM_INVALID, "[CheckReuse][MemoryOption]option %s=%s is invalid",
          OPTION_EXEC_DISABLE_REUSED_MEMORY, iter->second.c_str());
      REPORT_INNER_ERROR("E19999", "CheckReuseMemoryOption failed because option %s=%s is invalid.", 
      REPORT_INNER_ERROR("E19999", "CheckReuseMemoryOption failed because option %s=%s is invalid.",
          OPTION_EXEC_DISABLE_REUSED_MEMORY, iter->second.c_str());
      return FAILED;
    }
@@ -214,9 +213,6 @@ Status InnerSession::GetVariable(const std::string &name, Tensor &val) {

 Status InnerSession::AddGraph(uint32_t graph_id, const Graph &graph) {
  std::map<std::string, std::string> options;
  auto device_id = GetContext().DeviceId();
  GELOGD("Device id is %u", device_id);
  ProfilingManager::Instance().SetGraphIdToDeviceMap(graph_id, device_id);
  return AddGraph(graph_id, graph, options);
 }

@@ -248,8 +244,8 @@ Status InnerSession::AddGraphWithCopy(uint32_t graph_id, const Graph &graph,
  if (!init_flag_) {
    GELOGE(GE_SESS_INIT_FAILED, "[Add][Graph] failed because GraphManager not init, InnerSession:%lu, graph_id:%u.",
        session_id_, graph_id);
    REPORT_INNER_ERROR("E19999", 
        "AddGraphWithCopy failed because GraphManager not init, InnerSession:%lu, graph_id:%u.", 
    REPORT_INNER_ERROR("E19999",
        "AddGraphWithCopy failed because GraphManager not init, InnerSession:%lu, graph_id:%u.",
        session_id_, graph_id);
    return GE_SESS_INIT_FAILED;
  }
@@ -257,7 +253,7 @@ Status InnerSession::AddGraphWithCopy(uint32_t graph_id, const Graph &graph,
  Status ret = graph_manager_.AddGraphWithCopy(graph_id, graph, options, domi::GetContext());
  if (ret != SUCCESS) {
    GELOGE(ret, "[Add][Graph] failed, InnerSession:%lu graphid: %u.", session_id_, graph_id);
    REPORT_CALL_ERROR("E19999", 
    REPORT_CALL_ERROR("E19999",
        "GraphManager AddGraphWithCopy failed, InnerSession:%lu graphid: %u.", session_id_, graph_id);
    return ret;
  }
@@ -288,7 +284,7 @@ Status InnerSession::RunGraph(uint32_t graph_id, const std::vector<Tensor> &inpu
    domi::GetContext().user_out_nodes.clear();
    if (ret != SUCCESS) {
      GELOGE(ret, "[Run][Graph]failed, InnerSession:%lu graph_id=%u.", session_id_, graph_id);
      REPORT_CALL_ERROR("E19999", 
      REPORT_CALL_ERROR("E19999",
          "GraphManager RunGraph failed, InnerSession:%lu graph_id=%u.", session_id_, graph_id);
      return ret;
    }
@@ -528,7 +524,7 @@ Status InnerSession::SaveVariables(const Graph &graph, const std::vector<std::st
 Status InnerSession::AddDumpProperties(const DumpProperties &dump_properties) {
  if (!is_dump_server_inited_) {
    if (dump_properties.IsDumpOpen() || dump_properties.IsOpDebugOpen()) {
      GE_IF_BOOL_EXEC(AdxDataDumpServerInit() != kDumpStatus, 
      GE_IF_BOOL_EXEC(AdxDataDumpServerInit() != kDumpStatus,
                      GELOGE(PARAM_INVALID, "[Init][AdxDataDumpServer] failed, session_id:%lu.", session_id_);
                      return PARAM_INVALID)
      GELOGI("Init adx data dump server success");
--- a/inc/framework/executor/ge_executor.h
+++ b/inc/framework/executor/ge_executor.h
@@ -50,9 +50,26 @@ class GE_FUNC_VISIBILITY GeExecutor {
 public:
  GeExecutor();
  ~GeExecutor() = default;

  ge::Status Initialize();
  ge::Status Finalize();

  ///
  /// @ingroup ge
  /// @brief Initialize global execute environment.
  /// @param [in] options: environment variables.
  /// @return init result
  ///
  static Status Initialize(const std::map<std::string, std::string> &options);

  ///
  /// @ingroup ge
  /// @brief Finalize global execute environment.
  /// @param [in] model_id: model id allocate from manager
  /// @return execute result
  ///
  static Status FinalizeEx();

  ge::Status UnloadModel(uint32_t modelId);

  // Get input and output descriptor
--- a/inc/framework/omg/omg_inner_types.h
+++ b/inc/framework/omg/omg_inner_types.h
@@ -73,8 +73,7 @@ struct OMGBufferData {
 };

 struct OmgContext {
  OmgContext() { format = DOMI_TENSOR_ND; }
  domiTensorFormat_t format;
  domiTensorFormat_t format = DOMI_TENSOR_ND;

  // format of the input specified by the command line
  std::unordered_map<std::string, domiTensorFormat_t> input_nodes_format_map;
@@ -110,6 +109,11 @@ struct OmgContext {
  RunMode run_mode = ONLY_PRE_CHECK;
  bool train_flag = false;

  std::string forward_point;
  std::string backward_point;
  bool profiling_on = false;
  bool profiling_trace_on = false;

  std::string output_type;

  // Whether to use dynamic batch size or dynamic image size
--- a/tests/ut/ge/CMakeLists.txt
+++ b/tests/ut/ge/CMakeLists.txt
@@ -249,12 +249,7 @@ set(GRAPH_DAVINCI_MODEL_SRC_FILES
    "${GE_CODE_DIR}/ge/hybrid/node_executor/aicpu/aicpu_ext_info.cc"
 )

 set(GRAPH_EXECUTE_COMMON_SRC_FILES
    "${GE_CODE_DIR}/ge/hybrid/hybrid_davinci_model_stub.cc"
 )

 set(GRAPH_BUILD_COMMON_SRC_FILES
    "${GE_CODE_DIR}/ge/graph/manager/graph_manager.cc"
    "${GE_CODE_DIR}/ge/client/ge_api.cc"
    "${GE_CODE_DIR}/ge/session/inner_session.cc"
    "${GE_CODE_DIR}/ge/session/session_manager.cc"
@@ -263,6 +258,7 @@ set(GRAPH_BUILD_COMMON_SRC_FILES
    "${GE_CODE_DIR}/ge/plugin/engine/dnnengines.cc"
    "${GE_CODE_DIR}/ge/plugin/engine/engine_manage.cc"
    "${GE_CODE_DIR}/ge/graph/manager/graph_context.cc"
    "${GE_CODE_DIR}/ge/graph/manager/graph_manager.cc"
 )

 set(GRAPH_PASS_COMMON_SRC_FILES
@@ -857,25 +853,6 @@ target_link_libraries(ge_davinci_model PRIVATE
    json
 )

 # build graph execute common
 add_library(ge_execute_common STATIC ${GRAPH_EXECUTE_COMMON_SRC_FILES} ${PROTO_HDRS})

 target_compile_definitions(ge_execute_common PRIVATE
    google=ascend_private
 )

 target_compile_options(ge_execute_common PRIVATE
    -g --coverage -fprofile-arcs -ftest-coverage
    -Werror=format
 )

 target_link_libraries(ge_execute_common PRIVATE
    $<BUILD_INTERFACE:intf_pub>
    c_sec
    json
    ascend_protobuf
 )

 # build graph build common
 add_library(ge_build_common STATIC ${GRAPH_BUILD_COMMON_SRC_FILES} ${PROTO_HDRS})

@@ -955,7 +932,7 @@ target_compile_definitions(ut_libge_multiparts_utest PRIVATE
 target_link_libraries(ut_libge_multiparts_utest
    $<BUILD_INTERFACE:intf_pub>
    -Wl,--whole-archive
    ge_davinci_model ge_build_common ge_prepare_common ge_execute_common ge_pass_common ge_ut_common_format ge_ut_common
    ge_davinci_model ge_build_common ge_prepare_common ge_single_op ge_pass_common ge_ut_common_format ge_ut_common
    -Wl,--no-whole-archive
    gtest gtest_main gmock gmock_main ${COMMON_SHARED_LIBRARIES} -lrt -ldl -lgcov
 )
@@ -977,7 +954,7 @@ target_compile_options(ut_libge_others_utest PRIVATE
 target_link_libraries(ut_libge_others_utest
    $<BUILD_INTERFACE:intf_pub>
    -Wl,--whole-archive
    ge_davinci_model ge_build_common ge_prepare_common ge_pass_common ge_execute_common ge_ut_common ge_ut_common_format
    ge_davinci_model ge_build_common ge_prepare_common ge_pass_common ge_single_op ge_ut_common ge_ut_common_format
    -Wl,--no-whole-archive
    gtest gtest_main gmock gmock_main ${COMMON_SHARED_LIBRARIES} -lrt -ldl -lgcov
 )
@@ -997,7 +974,7 @@ target_compile_options(ut_libge_kernel_utest PRIVATE
 target_link_libraries(ut_libge_kernel_utest
    $<BUILD_INTERFACE:intf_pub>
    -Wl,--whole-archive
    ge_davinci_model ge_build_common ge_prepare_common ge_pass_common ge_execute_common ge_ut_common ge_ut_common_format
    ge_davinci_model ge_build_common ge_prepare_common ge_pass_common ge_single_op ge_ut_common ge_ut_common_format
    -Wl,--no-whole-archive
    gtest gtest_main gmock gmock_main ${COMMON_SHARED_LIBRARIES} -lrt -ldl -lgcov
 )
--- a/tests/ut/ge/profiling/ge_profiling_manager_unittest.cc
+++ b/tests/ut/ge/profiling/ge_profiling_manager_unittest.cc
@@ -31,8 +31,6 @@
 #include "graph/manager/graph_manager.h"
 #include "graph/ops_stub.h"
 #include "inc/framework/omg/omg_inner_types.h"
 #undef protected
 #undef private

 using namespace ge;
 using namespace std;
@@ -76,10 +74,21 @@ TEST_F(UtestGeProfilinganager, init_success) {

  struct MsprofGeOptions prof_conf = {{ 0 }};

  Status ret = ProfilingManager::Instance().InitFromOptions(options, prof_conf);
  Status ret = ProfilingManager::Instance().InitFromOptions(options.profiling_mode, options.profiling_options, options.job_id, prof_conf);
  EXPECT_EQ(ret, ge::SUCCESS);
 }

 TEST_F(UtestGeProfilinganager, initialize) {
  setenv("PROFILING_MODE", "true", true);

  const std::map<string, string> options({
      {OPTION_EXEC_JOB_ID, "0"}, {OPTION_EXEC_PROFILING_MODE, "1"}, {OPTION_EXEC_PROFILING_OPTIONS, R"({"result_path":"/data/profiling","training_trace":"on","task_trace":"on","aicpu_trace":"on","fp_point":"Data_0","bp_point":"addn","ai_core_metrics":"ResourceConflictRatio"})"}
  });

  ProfilingManager::Instance().Initialize(options);
  EXPECT_FALSE(ProfilingManager::Instance().is_execute_profiling_);
 }

 TEST_F(UtestGeProfilinganager, ParseOptions) {
  setenv("PROFILING_MODE", "true", true);
  Options options;
@@ -110,35 +119,35 @@ TEST_F(UtestGeProfilinganager, report_data_) {
  ProfilingManager::Instance().ReportData(0, data, tag_name);
 }

 TEST_F(UtestGeProfilinganager, get_fp_bp_point_) {
  map<std::string, string> options_map = {
    {OPTION_EXEC_PROFILING_OPTIONS,
    R"({"result_path":"/data/profiling","training_trace":"on","task_trace":"on","aicpu_trace":"on","fp_point":"Data_0","bp_point":"addn","ai_core_metrics":"ResourceConflictRatio"})"}};
  GEThreadLocalContext &context = GetThreadLocalContext();
  context.SetGraphOption(options_map);

  std::string fp_point;
  std::string bp_point;
  ProfilingManager::Instance().GetFpBpPoint(fp_point, bp_point);
  EXPECT_EQ(fp_point, "Data_0");
  EXPECT_EQ(bp_point, "addn");
 }

 TEST_F(UtestGeProfilinganager, get_fp_bp_point_empty) {
  // fp bp empty
  map<std::string, string> options_map = {
    { OPTION_EXEC_PROFILING_OPTIONS,
      R"({"result_path":"/data/profiling","training_trace":"on","task_trace":"on","aicpu_trace":"on","ai_core_metrics":"ResourceConflictRatio"})"}};
  GEThreadLocalContext &context = GetThreadLocalContext();
  context.SetGraphOption(options_map);
  std::string fp_point = "fp";
  std::string bp_point = "bp";
  ProfilingManager::Instance().bp_point_ = "";
  ProfilingManager::Instance().fp_point_ = "";
  ProfilingManager::Instance().GetFpBpPoint(fp_point, bp_point);
  EXPECT_EQ(fp_point, "");
  EXPECT_EQ(bp_point, "");
 }
 //TEST_F(UtestGeProfilinganager, get_fp_bp_point_) {
 //  map<std::string, string> options_map = {
 //    {OPTION_EXEC_PROFILING_OPTIONS,
 //    R"({"result_path":"/data/profiling","training_trace":"on","task_trace":"on","aicpu_trace":"on","fp_point":"Data_0","bp_point":"addn","ai_core_metrics":"ResourceConflictRatio"})"}};
 //  GEThreadLocalContext &context = GetThreadLocalContext();
 //  context.SetGraphOption(options_map);
 //
 //  std::string fp_point;
 //  std::string bp_point;
 //  ProfilingManager::Instance().GetFpBpPoint(fp_point, bp_point);
 //  EXPECT_EQ(fp_point, "Data_0");
 //  EXPECT_EQ(bp_point, "addn");
 //}

 //TEST_F(UtestGeProfilinganager, get_fp_bp_point_empty) {
 //  // fp bp empty
 //  map<std::string, string> options_map = {
 //    { OPTION_EXEC_PROFILING_OPTIONS,
 //      R"({"result_path":"/data/profiling","training_trace":"on","task_trace":"on","aicpu_trace":"on","ai_core_metrics":"ResourceConflictRatio"})"}};
 //  GEThreadLocalContext &context = GetThreadLocalContext();
 //  context.SetGraphOption(options_map);
 //  std::string fp_point = "fp";
 //  std::string bp_point = "bp";
 //  ProfilingManager::Instance().bp_point_ = "";
 //  ProfilingManager::Instance().fp_point_ = "";
 //  ProfilingManager::Instance().GetFpBpPoint(fp_point, bp_point);
 //  EXPECT_EQ(fp_point, "");
 //  EXPECT_EQ(bp_point, "");
 //}

 TEST_F(UtestGeProfilinganager, set_step_info_success) {
  uint64_t index_id = 0;