Online infer support dynamic dims

ge/common/types.cc

@@ -476,6 +476,9 @@ REGISTER_OPTYPE_DEFINE(HVDCALLBACKALLGATHER, "HorovodAllgather");
 REGISTER_OPTYPE_DEFINE(HVDCALLBACKBROADCAST, "HorovodBroadcast");
 REGISTER_OPTYPE_DEFINE(HVDWAIT, "HorovodWait");
 
+// aicpu op for online_infer dynamic_dims
+REGISTER_OPTYPE_DEFINE(GETDYNAMICDIMS, "GetDynamicDims");
+
 const std::string MODEL_ATTR_TASKS = "tasks";
 const std::string MODEL_ATTR_TASK_GEN_BASE_ADDR = "task_gen_base_addr";
 const std::string MODEL_ATTR_TASK_GEN_WEIGHT_ADDR = "task_gen_weight_addr";

ge/executor/CMakeLists.txt

@@ -61,6 +61,7 @@ set(SRC_LIST
     "../graph/load/new_model_manager/task_info/model_exit_task_info.cc"
     "../graph/load/new_model_manager/task_info/super_kernel/super_kernel_factory.cc"
     "../graph/load/new_model_manager/task_info/super_kernel/super_kernel.cc"
+    "../graph/common/local_context.cc"
     "../opskernel_manager/ops_kernel_builder_manager.cc"
     "../single_op/single_op_manager.cc"
     "../single_op/single_op_model.cc"

ge/executor/module.mk

@@ -63,6 +63,7 @@ local_ge_executor_src_files :=  \
     ../single_op/task/aicpu_kernel_task_builder.cc \
     ../hybrid/hybrid_davinci_model_stub.cc\
     ../hybrid/node_executor/aicpu/aicpu_ext_info.cc \
+    ../graph/common/local_context.cc \
 
 local_ge_executor_c_include :=             \
     proto/insert_op.proto                  \

ge/graph/load/new_model_manager/davinci_model.cc

@@ -62,6 +62,8 @@
 #include "runtime/rt_model.h"
 #include "runtime/stream.h"
 #include "securec.h"
+#include "graph/common/local_context.h"
+#include "common/formats/utils/formats_trans_utils.h"
 
 // create std::thread, catch exceptions using try/catch
 #define CREATE_STD_THREAD(thread_id, func, args)                                                  \
@@ -80,6 +82,7 @@ namespace {
 const uint32_t kDataIndex = 0;
 const uint32_t kOutputNum = 1;
 const uint32_t kTrueBranchStreamNum = 1;
+const uint32_t kGetDynamicDimsCount = 1;
 const uint32_t kThreadNum = 16;
 const uint32_t kAddrLen = sizeof(void *);
 const int kDecimal = 10;
@@ -88,6 +91,7 @@ const uint32_t kDataMemAlignSizeCompare = 64;
 const uint32_t kDumpL1FusionOpMByteSize = 2 * 1024 * 1024;
 const uint32_t kDumpFlagOfL1Fusion = 0;
 const char *const kDefaultBatchLable = "Batch_default";
+const char *const kGetDynamicDimsName = "ascend_mbatch_get_dynamic_dims_node";
 const int32_t kInvalidStream = -1;
 const uint32_t kEndOfSequence = 0x0704000a;
 const uint32_t kEndOfSequenceNew = 507005;
@@ -895,7 +899,6 @@ Status DavinciModel::InitNodes(const ComputeGraphPtr &compute_graph) {
     }
     GE_TIMESTAMP_ADD(InitTbeHandle);
   }
-
   AdjustDataOpList(data_by_index);
   GE_TIMESTAMP_CALLNUM_END(LoadTBEKernelBinToOpDesc, "GraphLoader::LoadTBEKernelBinToOpDesc.");
   GE_TIMESTAMP_CALLNUM_END(InitTbeHandle, "GraphLoader::InitTbeHandle.");
@@ -1051,6 +1054,15 @@ Status DavinciModel::InitInputZeroCopyNodes(const NodePtr &node) {
   return SUCCESS;
 }
 
+bool DavinciModel::IsGetNextSinkDynamic(const OpDescPtr &op_desc) {
+  bool getnext_sink_dynamic = false;
+  if (ge::AttrUtils::GetBool(op_desc, ATTR_GETNEXT_SINK_DYNMAIC, getnext_sink_dynamic) && getnext_sink_dynamic) {
+    GELOGI("ATTR_GETNEXT_SINK_DYNMAIC has been set and is true.");
+    return true;
+  }
+  return false;
+}
+
 /// @ingroup ge
 /// @brief NetOutput Op Initialize.
 /// @param [in] NodePtr: NetOutput Op.
@@ -1075,10 +1087,8 @@ Status DavinciModel::InitNetOutput(const NodePtr &node) {
   const vector<int64_t> input_size_list = ModelUtils::GetInputSize(op_desc);
   const vector<void *> virtual_addr_list = ModelUtils::GetInputDataAddrs(runtime_param_, op_desc);
   const vector<int64_t> input_offset_list = op_desc->GetInputOffset();
-  if (input_offset_list.size() != virtual_addr_list.size()) {
-    GELOGE(PARAM_INVALID, "virtual_addr size should be equal to offset size.");
-    return PARAM_INVALID;
-  }
+  GE_IF_BOOL_EXEC(input_offset_list.size() != virtual_addr_list.size(),
+                  GELOGE(PARAM_INVALID, "virtual_addr size should be equal to offset size."); return PARAM_INVALID;);
   if (input_size_list.empty() && virtual_addr_list.empty()) {
     GELOGI("NetOutput[%s] is empty.", op_desc->GetName().c_str());
     return SUCCESS;
@@ -1092,13 +1102,17 @@ Status DavinciModel::InitNetOutput(const NodePtr &node) {
   size_t num = new_output_data_info_.size();
   bool fusion_flag = false;
 
-  for (size_t idx = 0; idx < input_size_list.size(); ++idx) {
+  size_t input_count = input_size_list.size();
+  is_getnext_sink_dynamic_ = false;
+  if (IsGetNextSinkDynamic(op_desc)) {
+    input_count = input_size_list.size() - kGetDynamicDimsCount;
+    is_getnext_sink_dynamic_ = true;
+  }
+  for (size_t idx = 0; idx < input_count; ++idx) {
     ZeroCopyOffset zero_copy_offset;
     Status ret = zero_copy_offset.InitOutputDataInfo(input_size_list, virtual_addr_list, op_desc, idx, fusion_flag);
-    if (ret != SUCCESS) {
-      GELOGE(PARAM_INVALID, "InitDataInfo of input_info %s failed.", op_desc->GetName().c_str());
-      return PARAM_INVALID;
-    }
+    GE_IF_BOOL_EXEC(ret != SUCCESS, GELOGE(PARAM_INVALID, "InitDataInfo of input_info %s failed.",
+                                           op_desc->GetName().c_str()); return PARAM_INVALID;);
     new_output_data_info_[num + idx] = zero_copy_offset;
     void *addr = virtual_addr_list.at(idx);
     int64_t input_offset = input_offset_list.at(idx);
@@ -1118,14 +1132,204 @@ Status DavinciModel::InitNetOutput(const NodePtr &node) {
     GELOGI("SetOutputOutsideAddr success.");
   }
 
-  if (InitOutputZeroCopyNodes(node) != SUCCESS) {
-    GELOGE(PARAM_INVALID, "Output zero copy nodes init failed!");
-    return PARAM_INVALID;
+  GE_IF_BOOL_EXEC(InitOutputZeroCopyNodes(node) != SUCCESS,
+                  GELOGE(PARAM_INVALID, "Output zero copy nodes init failed!"); return PARAM_INVALID;);
+  GetAllGearsInfo(node);
+  if (is_getnext_sink_dynamic_) {
+    GE_IF_BOOL_EXEC(GetGetDynamicDimsNodeInfo(node) != SUCCESS,
+                    GELOGE(PARAM_INVALID, "Failed to get info of getdynamicdims node."); return PARAM_INVALID;);
   }
+  if (is_online_infer_dynamic_) {
+    GE_IF_BOOL_EXEC(GetGearAndRealOutSizeInfo(input_count, node) != SUCCESS,
+                    GELOGE(PARAM_INVALID, "Failed to get gear and real out size info."); return PARAM_INVALID;);
+    GE_IF_BOOL_EXEC(GetGearAndRealOutShapeInfo(input_count, op_desc) != SUCCESS,
+                    GELOGE(PARAM_INVALID, "Failed to get gear and real out shape info."); return PARAM_INVALID;);
+  }
+
   GELOGI("DavinciModel::InitNetoutput success.");
   return SUCCESS;
 }
 
+void DavinciModel::GetAllGearsInfo(const NodePtr &node) {
+  is_online_infer_dynamic_ = false;
+  all_gears_info_.clear();
+  std::string shapes;
+  (void) AttrUtils::GetStr(node->GetOpDesc(), ATTR_ALL_GEARS_INFO, shapes);
+  if (!shapes.empty()) {
+    is_online_infer_dynamic_ = true;
+    std::vector<std::string> shape_strs = ge::StringUtils::Split(shapes, ';');
+    for (const auto &shape_str : shape_strs) {
+      if (shape_str.empty()) {
+        continue;
+      }
+      std::vector<int64_t> gear_info;
+      std::vector<std::string> dims = ge::StringUtils::Split(shape_str, ',');
+      for (const auto &dim : dims) {
+        if (dim.empty()) {
+          continue;
+        }
+        gear_info.emplace_back(std::strtol(dim.c_str(), nullptr, kDecimal));
+      }
+      if (!gear_info.empty()) {
+        all_gears_info_.emplace_back(gear_info);
+        GELOGD("Init all gears info from %s, gaer info is %s.", node->GetName().c_str(),
+               formats::JoinToString(gear_info).c_str());
+      }
+    }
+  }
+}
+Status DavinciModel::GetGetDynamicDimsNodeInfo(const NodePtr &node) {
+  GE_CHECK_NOTNULL(node->GetOpDesc());
+  size_t input_count = node->GetAllInDataAnchors().size();
+  GELOGI("input_anchor count of %s is %zu.", node->GetName().c_str(), input_count);
+  size_t get_dynamic_dims_index = input_count - kGetDynamicDimsCount;
+  auto in_anchor = node->GetAllInDataAnchors().at(get_dynamic_dims_index);
+  auto peer_out_anchor = in_anchor->GetPeerOutAnchor();
+  if (peer_out_anchor == nullptr) {
+    GELOGE(PARAM_INVALID, "Out anchor of getdynmaicdims node should not be nullptr.");
+    return PARAM_INVALID;
+  }
+  auto peer_node = peer_out_anchor->GetOwnerNode();
+  auto op_desc = peer_node->GetOpDesc();
+  GE_CHECK_NOTNULL(op_desc);
+  if (op_desc->GetName() == kGetDynamicDimsName && op_desc->GetType() == GETDYNAMICDIMS) {
+    GELOGD("Start get info of %s.", op_desc->GetName().c_str());
+    auto input_addr = ModelUtils::GetInputDataAddrs(runtime_param_, node->GetOpDesc());
+    auto input_size = ModelUtils::GetInputSize(node->GetOpDesc());
+    if (input_addr.empty() || input_size.empty()) {
+      GELOGE(PARAM_INVALID, "Not set output of %s", op_desc->GetName().c_str());
+      return PARAM_INVALID;
+    }
+    auto input_desc = node->GetOpDesc()->GetInputDescPtr(get_dynamic_dims_index);
+    GE_CHECK_NOTNULL(input_desc);
+    if (input_desc->GetShape().GetDims().empty()) {
+      GELOGE(PARAM_INVALID, "Not set output desc shape of %s.", op_desc->GetName().c_str());
+      return PARAM_INVALID;
+    }
+    netoutput_last_input_addr_ = input_addr[get_dynamic_dims_index];
+    netoutput_last_input_size_ = input_size[get_dynamic_dims_index];
+    shape_of_cur_dynamic_dims_ = input_desc->GetShape().GetDims().at(0);
+    GELOGD("Shape of cur dynamic dims is %zu, size is %ld, addr is %p.", shape_of_cur_dynamic_dims_,
+           netoutput_last_input_size_, netoutput_last_input_addr_);
+  }
+  return SUCCESS;
+}
+
+Status DavinciModel::GetGearAndRealOutSizeInfo(size_t input_count, const NodePtr &node) {
+  GELOGD("Start get gear and real output size info of %s, input count is %zu.", node->GetName().c_str(), input_count);
+  merge_nodes_gear_and_real_out_size_info_.clear();
+  for (size_t idx = 0; idx < input_count; ++idx) {
+    auto in_anchor = node->GetAllInDataAnchors().at(idx);
+    auto peer_out_anchor = in_anchor->GetPeerOutAnchor();
+    if (peer_out_anchor == nullptr) {
+      continue;
+    }
+    auto peer_node = peer_out_anchor->GetOwnerNode();
+    auto op_desc = peer_node->GetOpDesc();
+    GE_CHECK_NOTNULL(op_desc);
+    if ((peer_node->GetType() == MERGE) && (op_desc->HasAttr(ATTR_INSERT_BY_MBATCH))) {
+      if (GetRealOutputSizeOfMerge(idx, peer_node) != SUCCESS) {
+        GELOGE(PARAM_INVALID, "Get real output size of %s failed.", peer_node->GetName().c_str());
+        return PARAM_INVALID;
+      }
+    }
+  }
+  return SUCCESS;
+}
+
+Status DavinciModel::GetRealOutputSizeOfMerge(size_t input_index, const NodePtr &merge_node) {
+  GELOGD("Start get output size of %s, which is %zu input to netoutput.", merge_node->GetName().c_str(), input_index);
+  std::map<vector<int64_t>, int64_t> gear_and_real_out_size_info;
+  for (auto &in_anchor : merge_node->GetAllInDataAnchors()) {
+    auto peer_out_anchor = in_anchor->GetPeerOutAnchor();
+    if (peer_out_anchor == nullptr) {
+      continue;
+    }
+    auto in_node = peer_out_anchor->GetOwnerNode();
+    GELOGD("Input node of merge is %s.", in_node->GetName().c_str());
+    auto op_desc = in_node->GetOpDesc();
+    GE_CHECK_NOTNULL(op_desc);
+    string batch_label;
+    if (AttrUtils::GetStr(op_desc, ATTR_NAME_BATCH_LABEL, batch_label)) {
+      size_t batch_index = static_cast<size_t>(stoi(batch_label.substr(batch_label.rfind('_') + 1)));
+      GELOGD("Batch index of %s is %zu.", op_desc->GetName().c_str(), batch_index);
+      if (batch_index > all_gears_info_.size()) {
+        GELOGE(PARAM_INVALID, "The value of ATTR_NAME_BATCH_LABEL is invalid.");
+        return PARAM_INVALID;
+      }
+
+      const vector<int64_t> output_size_list = ModelUtils::GetOutputSize(op_desc);
+      int output_index = ge::AnchorUtils::GetIdx(peer_out_anchor);
+      auto tensor_desc = op_desc->GetOutputDescPtr(output_index);
+      GE_CHECK_NOTNULL(tensor_desc);
+      int64_t data_size = 0;
+      if (TensorUtils::GetTensorSizeInBytes(*tensor_desc, data_size) != GRAPH_SUCCESS) {
+        GELOGE(FAILED, "Get tensor size in bytes failed.");
+        return FAILED;
+      }
+      gear_and_real_out_size_info[all_gears_info_[batch_index]] = data_size;
+      GELOGD("Get real gear index is: %zu, gear info is %s, size is %ld, tensor size is %ld",
+             batch_index, formats::JoinToString(all_gears_info_[batch_index]).c_str(),
+             output_size_list[output_index], data_size);
+    }
+  }
+  merge_nodes_gear_and_real_out_size_info_[input_index] = gear_and_real_out_size_info;
+  return SUCCESS;
+}
+
+Status DavinciModel::GetGearAndRealOutShapeInfo(size_t input_count, const OpDescPtr &op_desc) {
+  GELOGD("Start to get dynamic output dims of %s.", op_desc->GetName().c_str());
+  merge_nodes_gear_and_real_out_shape_info_.clear();
+  std::vector<std::string> dynamic_output_shape_info;
+  if (!AttrUtils::GetListStr(op_desc, ATTR_NAME_DYNAMIC_OUTPUT_DIMS, dynamic_output_shape_info)) {
+    GELOGD("Can not get dynamic output dims attr");
+    return SUCCESS;
+  }
+  GELOGI("Dynamic output shape info is %s", formats::JoinToString(dynamic_output_shape_info).c_str());
+  std::vector<vector<int64_t>> dynamic_output_shape;
+  ParseDynamicOutShape(dynamic_output_shape_info, dynamic_output_shape);
+  // idx: input_index to netoutput
+  for (size_t idx = 0; idx < input_count; ++idx) {
+    std::map<vector<int64_t>, vector<int64_t>> gear_and_real_out_shape_info;
+    for (auto &it : dynamic_output_shape) {
+      auto gear_index = static_cast<size_t>(it[0]);
+      if (gear_index > all_gears_info_.size()) {
+        GELOGE(PARAM_INVALID, "The value of cur index: %zu is invalid.", static_cast<size_t>(it[0]));
+        return PARAM_INVALID;
+      }
+
+      if (static_cast<size_t>(it[1]) == idx) {
+        vector<int64_t> output_shape;
+        for (size_t i = 2; i < it.size(); ++i) {
+          output_shape.emplace_back(it[i]);
+        }
+        gear_and_real_out_shape_info[all_gears_info_[gear_index]] = output_shape;
+        GELOGD("Get real gear index is: %zu, gear info is %s, output shape is %s.",
+               gear_index, formats::JoinToString(all_gears_info_[gear_index]).c_str(),
+               formats::JoinToString(output_shape).c_str());
+      }
+    }
+    merge_nodes_gear_and_real_out_shape_info_[idx] = gear_and_real_out_shape_info;
+  }
+  return SUCCESS;
+}
+
+void DavinciModel::ParseDynamicOutShape(const std::vector<std::string> &str_info,
+                                        std::vector<vector<int64_t>> &vec_info) {
+  for (size_t i = 0; i < str_info.size(); ++i) {
+    std::vector<int64_t> shape;
+    std::vector<std::string> dims = ge::StringUtils::Split(str_info[i], ',');
+    for (const auto &dim : dims) {
+      if (dim.empty()) {
+        continue;
+      }
+      shape.emplace_back(std::strtol(dim.c_str(), nullptr, kDecimal));
+    }
+    GELOGI("Shape from attr is %s.", formats::JoinToString(shape).c_str());
+    vec_info.emplace_back(shape);
+  }
+}
+
 ///
 /// @ingroup ge
 /// @brief output zero copy node Initialize.
@@ -2324,6 +2528,7 @@ Status DavinciModel::CopyOutputData(uint32_t data_id, OutputData &output_data, r
   }
 
   std::vector<DataBuffer> &blobs = output_data.blobs;
+  size_t idx = 0;
   for (const auto &output : new_output_data_info_) {
     if (output.first >= blobs.size()) {
       GELOGE(FAILED, "Blobs not match: blobs=%zu, tensor=%zu, index=%u, size=%ld", blobs.size(),
@@ -2349,13 +2554,19 @@ Status DavinciModel::CopyOutputData(uint32_t data_id, OutputData &output_data, r
     } else if (buffer.length > mem_size) {
       GELOGW("Tensor data size=%lu, buffer size=%u", mem_size, buffer.length);
     }
-    uint64_t data_size = output.second.GetDataSize();
+    int64_t data_size = output.second.GetDataSize();
+
+    if (is_online_infer_dynamic_) {
+      auto gear_and_real_out_size_info = merge_nodes_gear_and_real_out_size_info_[idx];
+      data_size = gear_and_real_out_size_info[cur_dynamic_dims_];
+    }
     uint64_t buffer_length = buffer.length;
     void *buffer_addr = reinterpret_cast<void *>(reinterpret_cast<uintptr_t>(buffer.data));
 
-    GELOGI("[IMAS]CopyPlainData memcpy graph_%u type[F] output[%u] memaddr[%p] mem_size[%lu] datasize[%u]",
+    GELOGI("[IMAS]CopyPlainData memcpy graph_%u type[F] output[%u] memaddr[%p] mem_size[%ld] datasize[%u]",
            runtime_param_.graph_id, output.first, output.second.GetBasicAddr(), data_size, buffer_length);
     GE_CHK_RT_RET(rtMemcpy(buffer_addr, buffer_length, output.second.GetBasicAddr(), data_size, kind));
+    idx++;
   }
   return SUCCESS;
 }
@@ -2371,19 +2582,29 @@ Status DavinciModel::GenOutputTensorInfo(const OpDescPtr &op_desc, uint32_t data
   std::vector<int64_t> out_buffer_size_vec;
   std::vector<std::vector<int64_t>> shape_info_vec;
   size_t input_num = op_desc->GetInputsSize();
+  if (is_getnext_sink_dynamic_) {
+    input_num = input_num - kGetDynamicDimsCount;
+  }
   for (size_t i = 0; i < input_num; ++i) {
     int64_t size = 0;
     auto input_desc = op_desc->GetInputDescPtr(i);
     GE_CHECK_NOTNULL(input_desc);
     auto ret = TensorUtils::GetTensorSizeInBytes(*input_desc, size);
-    if (ret != GRAPH_SUCCESS) {
-      GELOGE(ret, "Get size from TensorDesc failed, op:%s, input index:%zu", op_desc->GetName().c_str(), i);
-      return ret;
+    GE_IF_BOOL_EXEC(ret != GRAPH_SUCCESS,
+                    GELOGE(ret, "Get size from TensorDesc failed, op:%s, input id:%zu", op_desc->GetName().c_str(), i);
+                    return ret);
+    std::vector<int64_t> output_shape = input_desc->GetShape().GetDims();
+    if (is_online_infer_dynamic_) {
+      auto gear_and_real_out_size_info = merge_nodes_gear_and_real_out_size_info_[i];
+      size = gear_and_real_out_size_info[cur_dynamic_dims_];
+      auto gear_and_real_out_shape_info = merge_nodes_gear_and_real_out_shape_info_[i];
+      output_shape = gear_and_real_out_shape_info[cur_dynamic_dims_];
+      is_dynamic_ = true;
     }
+    GELOGI("Output size is %ld, output shape is %s.", size, formats::JoinToString(output_shape).c_str());
     out_buffer_size_vec.push_back(size);
-    shape_info_vec.push_back(input_desc->GetShape().GetDims());
+    shape_info_vec.push_back(output_shape);
   }
-
   GELOGI("Output blobs size:%zu, data index:%u, model id:%u", out_buffer_size_vec.size(), data_index, model_id_);
   for (size_t i = 0; i < out_buffer_size_vec.size(); ++i) {
     std::unique_ptr<uint8_t[]> data_buf(new (std::nothrow) uint8_t[out_buffer_size_vec[i]]);
@@ -2397,7 +2618,8 @@ Status DavinciModel::GenOutputTensorInfo(const OpDescPtr &op_desc, uint32_t data
     output.data = std::move(data_buf);
     output.length = out_buffer_size_vec[i];
     outputs.emplace_back(std::move(output));
-    GELOGI("Output index:%zu, data_length:%lu.", i, output.length);
+    GELOGD("Output index:%zu, output dims is %s, data length:%lu.", i,
+           formats::JoinToString(output.dims).c_str(), output.length);
   }
   return SUCCESS;
 }
@@ -2446,8 +2668,18 @@ Status DavinciModel::ReturnResult(uint32_t data_id, const bool rslt_flg, const b
   output_data->index = data_id;
   output_data->model_id = model_id_;
 
+  is_getnext_sink_dynamic_ = false;
   // copy output data from op to designated position
   for (auto &op_desc : output_op_list_) {
+    if (IsGetNextSinkDynamic(op_desc)) {
+      GELOGD("Reinit cur dynamic dims when getnext sink dynamic.");
+      is_getnext_sink_dynamic_ = true;
+      cur_dynamic_dims_.clear();
+      cur_dynamic_dims_.resize(shape_of_cur_dynamic_dims_);
+      GE_CHK_RT_RET(rtMemcpy(cur_dynamic_dims_.data(), shape_of_cur_dynamic_dims_ * sizeof(int64_t),
+                                netoutput_last_input_addr_, netoutput_last_input_size_, RT_MEMCPY_DEVICE_TO_HOST));
+    }
+    GELOGD("Cur dynamic dims is %s.", formats::JoinToString(cur_dynamic_dims_).c_str());
     if (GenOutputTensorInfo(op_desc, data_index, output_data, outputs) != SUCCESS) {
       return INTERNAL_ERROR;
     }
@@ -2520,12 +2752,20 @@ void *DavinciModel::Run(DavinciModel *model) {
       continue;
     }
     GELOGI("Getting the input data, model_id:%u", model_id);
-
     GE_IF_BOOL_EXEC(!model->RunFlag(), break);
 
     InputData current_data = data_wrapper->GetInput();
     GELOGI("Model thread Run begin, model id:%u, data index:%u.", model_id, current_data.index);
-
+    if (model->is_online_infer_dynamic_ && !model->is_getnext_sink_dynamic_) {
+      model->cur_dynamic_dims_.clear();
+      GE_IF_BOOL_EXEC(current_data.blobs.empty(), break);
+      auto shape_data_buffer_data = current_data.blobs.back().data;
+      auto shape_data_buffer_length = current_data.blobs.back().length;
+      model->cur_dynamic_dims_.assign(reinterpret_cast<int64_t *>(shape_data_buffer_data),
+                                      reinterpret_cast<int64_t *>(shape_data_buffer_data) +
+                                      shape_data_buffer_length / sizeof(int64_t));
+      GELOGD("Data: cur dynamic dims is %s", formats::JoinToString(model->cur_dynamic_dims_).c_str());
+    }
     GE_TIMESTAMP_START(Model_SyncVarData);
     ret = model->SyncVarData();
     GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(

ge/graph/load/new_model_manager/davinci_model.h

@@ -47,6 +47,7 @@
 #include "mmpa/mmpa_api.h"
 #include "proto/task.pb.h"
 #include "task_info/task_info.h"
+#include "graph/common/local_context.h"
 
 namespace ge {
 // op debug need 2048 bits buffer
@@ -523,7 +524,6 @@ class DavinciModel {
   bool is_inner_p2p_mem_base_;
   // input data manager
   DataInputer *data_inputer_;
-
   int64_t load_begin_time_;
   int64_t load_end_time_;
   struct timeInfo time_info_;
@@ -842,6 +842,14 @@ class DavinciModel {
   void ParseAIPPInfo(std::string in_out_info, InputOutputDims &dims_info);
   void SetLabelForDynamic(const NodePtr &node);
 
+  void ParseDynamicOutShape(const std::vector<std::string> &str_info, std::vector<vector<int64_t>> &vec_info);
+  bool IsGetNextSinkDynamic(const OpDescPtr &op_desc);
+  void GetAllGearsInfo(const NodePtr &node);
+  Status GetGetDynamicDimsNodeInfo(const NodePtr &node);
+  Status GetGearAndRealOutSizeInfo(size_t input_count, const NodePtr &node);
+  Status GetRealOutputSizeOfMerge(size_t input_index, const NodePtr &merge_node);
+  Status GetGearAndRealOutShapeInfo(size_t input_count, const OpDescPtr &op_desc);
+
   bool is_model_has_inited_;
   uint32_t model_id_;
   uint32_t runtime_model_id_;
@@ -994,6 +1002,17 @@ class DavinciModel {
   void *op_debug_addr_ = nullptr;
   void *p2p_debug_addr_ = nullptr;
   bool is_new_model_desc_{false};
+  bool is_online_infer_dynamic_ = false;
+  bool is_getnext_sink_dynamic_ = false;
+  std::vector<int64_t> cur_dynamic_dims_;
+  void *netoutput_last_input_addr_ = nullptr;
+  int64_t netoutput_last_input_size_ = 0;
+  size_t shape_of_cur_dynamic_dims_ = 0;
+  // key: input_index: input is merge node; value: each gear info and each output size
+  std::map<size_t, std::map<vector<int64_t>, int64_t>> merge_nodes_gear_and_real_out_size_info_;
+  // key: input_index: input is merge node; value: each gear info and each output shape
+  std::map<size_t, std::map<vector<int64_t>, vector<int64_t>>> merge_nodes_gear_and_real_out_shape_info_;
+  std::vector<std::vector<int64_t>> all_gears_info_;
 };
 }  // namespace ge
 #endif  // GE_GRAPH_LOAD_NEW_MODEL_MANAGER_DAVINCI_MODEL_H_

ge/graph/load/new_model_manager/model_manager.cc

@@ -29,6 +29,8 @@
 #include "graph/load/new_model_manager/davinci_model.h"
 #include "graph/load/new_model_manager/davinci_model_parser.h"
 #include "model/ge_root_model.h"
+#include "graph/common/local_context.h"
+#include "common/formats/utils/formats_trans_utils.h"
 
 namespace ge {
 thread_local uint32_t device_count = 0;
@@ -405,6 +407,10 @@ Status ModelManager::Unload(uint32_t model_id) {
   }
   std::lock_guard<std::mutex> lock(exeception_infos_mutex_);
   exception_infos_.clear();
+  for (auto addr : shape_data_addrs_[model_id]) {
+    delete[] addr;
+  }
+  shape_data_addrs_.erase(model_id);
   return SUCCESS;
 }
 
@@ -444,6 +450,34 @@ Status ModelManager::DataInput(const InputData &input_data, OutputData &output_d
   return SUCCESS;
 }
 
+Status ModelManager::GetCurDynamicDims(const vector<vector<int64_t>> &user_real_input_dims,
+                                       const vector<pair<string, vector<int64_t>>> &user_input_dims,
+                                       vector<int64_t> &cur_dynamic_dims) {
+  GELOGD(" Start get cur dynamic dims.");
+  if (user_real_input_dims.size() != user_input_dims.size()) {
+    GELOGE(INTERNAL_ERROR,
+           "The input count of user: %zu should be equal to the data count of graph: %zu",
+           user_real_input_dims.size(), user_input_dims.size());
+    return INTERNAL_ERROR;
+  }
+
+  for (size_t i = 0; i < user_input_dims.size(); ++i) {
+    if (user_real_input_dims[i].size() != user_input_dims[i].second.size()) {
+      GELOGE(INTERNAL_ERROR,
+             "The shape size: %zu of dynamic input: %s should be equal to the shape size of input shape: %zu.",
+             user_real_input_dims[i].size(), user_input_dims[i].first.c_str(), user_input_dims[i].second.size());
+      return INTERNAL_ERROR;
+    }
+    for (size_t j = 0; j < user_input_dims.at(i).second.size(); ++j) {
+      if (user_input_dims.at(i).second.at(j) < 0) {
+        cur_dynamic_dims.emplace_back(user_real_input_dims[i][j]);
+      }
+    }
+  }
+  GELOGD("Cur dynamic dims is %s.", formats::JoinToString(cur_dynamic_dims).c_str());
+  return SUCCESS;
+}
+
 ///
 /// @ingroup domi_ome
 /// @brief load Input and output TensorInfo for Model
@@ -461,13 +495,31 @@ Status ModelManager::DataInputTensor(uint32_t model_id, const std::vector<InputT
   input_data.timeout = 0;
   input_data.timestamp = 0;
   input_data.index = 0;
-
   for (size_t i = 0; i < inputs.size(); ++i) {
     DataBuffer data;
     data.data = inputs[i].data;
     data.length = inputs[i].length;
     input_data.blobs.push_back(data);
   }
+  if (!GetLocalOmgContext().user_input_dims.empty() && GetLocalOmgContext().need_multi_batch) {
+    std::vector<int64_t> cur_dynamic_dims;
+    if (!GetLocalOmgContext().user_real_input_dims.empty()) {
+      if (GetCurDynamicDims(GetLocalOmgContext().user_real_input_dims, GetLocalOmgContext().user_input_dims,
+                            cur_dynamic_dims) != SUCCESS) {
+        GELOGE(INTERNAL_ERROR, "[Train_Dynamic] Failed to Parse real_dynamic_dims.");
+        return INTERNAL_ERROR;
+      }
+      DataBuffer data;
+      data.data = new(std::nothrow) int64_t[cur_dynamic_dims.size()];
+      GE_CHECK_NOTNULL(data.data);
+      uint64_t length = static_cast<uint64_t>(cur_dynamic_dims.size() * sizeof(int64_t));
+      GE_CHK_BOOL_EXEC(memcpy_s(data.data, length, cur_dynamic_dims.data(), length) == EOK, return INTERNAL_ERROR,
+                       "Failed to memcpy data.");
+      data.length = length;
+      input_data.blobs.push_back(data);
+      shape_data_addrs_[model_id].emplace_back(reinterpret_cast<int64_t *>(data.data));
+    }
+  }
 
   OutputData output_data;
   output_data.model_id = model_id;

ge/graph/load/new_model_manager/model_manager.h

@@ -124,6 +124,18 @@ class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ModelManager {
 
   ge::Status DataInputTensor(uint32_t model_id, const std::vector<InputTensorInfo> &inputs);
 
+  ///
+  /// @ingroup domi_ome
+  /// @brief Get cur_dynamic_dims for all input.
+  /// @param [in] vector<vector<uint64_t>> &user_real_input_dims: dims info of all user_inputs.
+  /// @param [in] vector<pair<string, vector<int64_t>>> &user_input_dims: key:name. value:dynamic dims from option.
+  /// @param [out] vector<uint64_t> &cur_dynamic_dims: real dims gather, where the index of -1.
+  /// @return 0: SUCCESS / others: INTERNAL_ERROR
+  ///
+  Status GetCurDynamicDims(const vector<vector<int64_t>> &user_real_input_dims,
+                           const vector<pair<string, vector<int64_t>>> &user_input_dims,
+                           vector<int64_t> &cur_dynamic_dims);
+
   ///
   /// @ingroup domi_ome
   /// @brief model start to run
@@ -352,6 +364,7 @@ class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ModelManager {
   std::map<uintptr_t, std::map<std::string, CustAICPUKernelPtr>> cust_aicpu_so_;
 
   static DumpProperties dump_properties_;
+  std::map<uint32_t, std::vector<int64_t *>> shape_data_addrs_;
 };
 }  // namespace ge
 

ge/graph/manager/graph_manager.cc

@@ -108,6 +108,10 @@
 #include "graph/types.h"
 #include "inc/pass_manager.h"
 #include "init/gelib.h"
+#include "ir_build/atc_ir_common.h"
+#include "graph/common/local_context.h"
+#include "graph/common/omg_util.h"
+#include "common/formats/utils/formats_trans_utils.h"
 
 namespace {
 const char *const kSummary = "Summary";
@@ -121,6 +125,12 @@ const char *const kCheckPointGraph = "checkpoint_graph";
 const char *const kVectorEngine = "VectorEngine";
 const char *const kAIcoreEngine = "AIcoreEngine";
 const char *const kOffOptimize = "off_optimize";
+const int32_t kDynamicDimsTypeIsGetNext = 0;
+const int32_t kDynamicDimsTypeIsData = 1;
+const int64_t kInvalidDynaimcDimsType = -1;
+const char *const kSubstrOfGetNextNosinkName = "IteratorGetNext";
+const char *const kShapeDataName = "ascend_mbatch_shape_data";
+const char *const kGetNextName = "IteratorV2";
 
 bool IsTailingOptimization() {
   string is_tailing_optimization_option;
@@ -284,6 +294,42 @@ Status GraphManager::Finalize() {
   return unload_model_ret;
 }
 
+Status GraphManager::InitDynamicParams(ComputeGraphPtr &compute_graph) {
+  for (const auto &node : compute_graph->GetAllNodes()) {
+    auto op_desc = node->GetOpDesc();
+    if (op_desc == nullptr) {
+      continue;
+    }
+    GetLocalOmgContext().need_multi_batch = false;
+    std::string op_type;
+    auto ret = GetOriginalType(node, op_type);
+    if (ret != SUCCESS) {
+      GELOGE(FAILED, "Failed to get node %s original type.", node->GetName().c_str());
+      return FAILED;
+    }
+    if ((op_desc->GetType() == DATA) || (op_type == kGetNextName)) {
+      GELOGI("Need to process multi batch for compute graph.");
+      GetLocalOmgContext().need_multi_batch = true;
+      break;
+    }
+  }
+  if (!options_.input_shape.empty() && !options_.dynamic_dims.empty()) {
+    if (!ge::ParseInputShape(options_.input_shape, GetLocalOmgContext().input_dims,
+                             GetLocalOmgContext().user_input_dims, true)) {
+      GELOGE(GRAPH_PARAM_INVALID, "Failed to parse input shape: %s.", options_.input_shape.c_str());
+      return GRAPH_PARAM_INVALID;
+    }
+    GetLocalOmgContext().dynamic_dims = options_.dynamic_dims;
+  }
+  if (options_.dynamic_node_type == kDynamicDimsTypeIsGetNext) {
+    GetLocalOmgContext().dynamic_node_type = GETNEXT;
+  }
+  if (options_.dynamic_node_type == kDynamicDimsTypeIsData) {
+    GetLocalOmgContext().dynamic_node_type = DATA;
+  }
+  return SUCCESS;
+}
+
 Status GraphManager::AddGraph(const GraphId &graph_id, const Graph &graph,
                               const std::map<std::string, std::string> &options,
                               const OmgContext &omg_context) {
@@ -303,6 +349,7 @@ Status GraphManager::AddGraph(const GraphId &graph_id, const Graph &graph,
       return GE_GRAPH_GRAPH_ALREADY_EXIST;
     }
     (void)AttrUtils::SetBool(*compute_graph, ATTR_NAME_GRAPH_HAS_BEEN_ADDED, true);
+    compute_graph_ = compute_graph;
   } else {
     GELOGE(FAILED, "compute graph is null");
     return FAILED;
@@ -320,15 +367,11 @@ Status GraphManager::AddGraph(const GraphId &graph_id, const Graph &graph,
   }
 
   GraphNodePtr graph_node = MakeShared<ge::GraphNode>(graph_id);
-  if (graph_node == nullptr) {
-    GELOGE(FAILED, "GraphNode make shared failed");
-    return FAILED;
-  }
+  GE_IF_BOOL_EXEC(graph_node == nullptr, GELOGE(FAILED, "GraphNode make shared failed");
+                  return FAILED);
   std::shared_ptr<Graph> graph_ptr = MakeShared<ge::Graph>(graph);
-  if (graph_ptr == nullptr) {
-    GELOGE(FAILED, "GraphPtr make shared failed");
-    return FAILED;
-  }
+  GE_IF_BOOL_EXEC(graph_ptr == nullptr, GELOGE(FAILED, "GraphPtr make shared failed");
+                  return FAILED);
 
   graph_node->SetGraph(graph_ptr);
   graph_node->SetOptions(options);
@@ -338,6 +381,10 @@ Status GraphManager::AddGraph(const GraphId &graph_id, const Graph &graph,
   if (!options_.output_datatype.empty()) {
     GetLocalOmgContext().output_type = options_.output_datatype;
   }
+  if (InitDynamicParams(compute_graph) != SUCCESS) {
+    GELOGE(GRAPH_PARAM_INVALID, "Failed to init params when online infer is dynamic.");
+    return GRAPH_PARAM_INVALID;
+  }
 
   CompilerStages &stages = GetCompilerStages(graph_id);
   stages.preparer.SetOptions(options_);
@@ -1402,10 +1449,9 @@ Status GraphManager::ParseOptions(const std::map<std::string, std::string> &opti
 
   // get encrypt mode
   ret = ParseOption(options, ENCRYPT_MODE, options_.encrypt_mode);
-  if (ret != SUCCESS) {
-    GELOGE(GE_GRAPH_OPTIONS_INVALID, "Key:ge.encryptMode value invalid.");
-    return GE_GRAPH_OPTIONS_INVALID;
-  }
+  GE_IF_BOOL_EXEC(ret != SUCCESS,
+                  GELOGE(GE_GRAPH_OPTIONS_INVALID, "Key:ge.encryptMode value invalid.");
+                  return GE_GRAPH_OPTIONS_INVALID);
 
   // get ek file
   ParseOption(options, EK_FILE, options_.ek_file);
@@ -1443,33 +1489,29 @@ Status GraphManager::ParseOptions(const std::map<std::string, std::string> &opti
 
   // get weight compress flag
   ret = ParseOption(options, COMPRESS_FLAG, options_.compress_flag);
-  if (ret != SUCCESS) {
-    GELOGE(GE_GRAPH_OPTIONS_INVALID, "Key:ge.compressFlag value is invalid, must be 0 or 1.");
-    return GE_GRAPH_OPTIONS_INVALID;
-  }
+  GE_IF_BOOL_EXEC(ret != SUCCESS,
+                  GELOGE(GE_GRAPH_OPTIONS_INVALID, "Key:ge.compressFlag value is invalid, must be 0 or 1.");
+                  return GE_GRAPH_OPTIONS_INVALID);
 
   // ge.graphType.
   options_.run_graph_flag = true;
   ret = ParseOption(options, RUN_FLAG, options_.run_graph_flag);
-  if (ret != SUCCESS) {
-    GELOGE(GE_GRAPH_OPTIONS_INVALID, "Key:ge.runFlag value is invalid, must be 0 or 1.");
-    return GE_GRAPH_OPTIONS_INVALID;
-  }
+  GE_IF_BOOL_EXEC(ret != SUCCESS,
+                  GELOGE(GE_GRAPH_OPTIONS_INVALID, "Key:ge.runFlag value is invalid, must be 0 or 1.");
+                  return GE_GRAPH_OPTIONS_INVALID);
 
   // ge.graphType
   ret = ParseTrainGraphFlag(options_.run_graph_flag, options_.train_graph_flag);
-  if (ret != SUCCESS) {
-    GELOGE(GE_GRAPH_OPTIONS_INVALID, "Key:ge.runFlag value is invalid");
-    return GE_GRAPH_OPTIONS_INVALID;
-  }
+  GE_IF_BOOL_EXEC(ret != SUCCESS,
+                  GELOGE(GE_GRAPH_OPTIONS_INVALID, "Key:ge.runFlag value is invalid");
+                  return GE_GRAPH_OPTIONS_INVALID);
 
   // parse FmkOp
   options_.local_fmk_op_flag = false;
   ret = ParseOption(options, LOCAL_FMKOP_FLAG, options_.local_fmk_op_flag);
-  if (ret != SUCCESS) {
-    GELOGE(GE_GRAPH_OPTIONS_INVALID, "Key:ge.localFmkopFlag value is invalid, must be 0 or 1.");
-    return GE_GRAPH_OPTIONS_INVALID;
-  }
+  GE_IF_BOOL_EXEC(ret != SUCCESS,
+                  GELOGE(GE_GRAPH_OPTIONS_INVALID, "Key:ge.localFmkopFlag value is invalid, must be 0 or 1.");
+                  return GE_GRAPH_OPTIONS_INVALID);
   options_.enable_print_op_pass = true;
   ret = ParseOption(options, ENABLE_PRINT_OP_PASS, options_.enable_print_op_pass);
 
@@ -1481,11 +1523,9 @@ Status GraphManager::ParseOptions(const std::map<std::string, std::string> &opti
   // parse hcom parallel
   options_.hcom_parallel = false;
   ret = ParseOption(options, HCOM_PARALLEL, options_.hcom_parallel);
-  if (ret != SUCCESS) {
-    GELOGE(GE_GRAPH_OPTIONS_INVALID, "Key:ge.hcomParallel value is invalid, must be 0 or 1.");
-    return GE_GRAPH_OPTIONS_INVALID;
-  }
-
+  GE_IF_BOOL_EXEC(ret != SUCCESS,
+                  GELOGE(GE_GRAPH_OPTIONS_INVALID, "Key:ge.hcomParallel value is invalid, must be 0 or 1.");
+                  return GE_GRAPH_OPTIONS_INVALID);
   // net output node dataType
   ParseOption(options, OUTPUT_DATATYPE, options_.output_datatype);
 
@@ -1495,6 +1535,22 @@ Status GraphManager::ParseOptions(const std::map<std::string, std::string> &opti
   // Original model file name
   ParseOption(options, ORIGINAL_MODEL_FILE, options_.original_model_file);
 
+  ParseOption(options, INPUT_SHAPE, options_.input_shape);
+  ParseOption(options, kDynamicDims, options_.dynamic_dims);
+  ParseOption(options, DYNAMIC_NODE_TYPE, options_.dynamic_node_type);
+  GELOGD("Dynamic dims params: input shape is %s, dynamic dims is %s, dynamic node type is %d.",
+         options_.input_shape.c_str(), options_.dynamic_dims.c_str(), options_.dynamic_node_type);
+  if ((!options_.input_shape.empty() && options_.dynamic_dims.empty()) ||
+      (options_.input_shape.empty() && !options_.dynamic_dims.empty())) {
+      GELOGE(GRAPH_PARAM_INVALID, "Should set input shape and dynamic dims at the same time");
+      return GRAPH_PARAM_INVALID;
+  }
+  if ((!options_.input_shape.empty() && options_.dynamic_node_type == kInvalidDynaimcDimsType) ||
+      (!options_.dynamic_dims.empty() && options_.dynamic_node_type == kInvalidDynaimcDimsType)) {
+    GELOGE(GRAPH_PARAM_INVALID, "Should set valid dynamic node type");
+    return GRAPH_PARAM_INVALID;
+  }
+
   // Set Build model and step
   ParseOption(options, BUILD_MODE, options_.build_mode);
   ParseOption(options, BUILD_STEP, options_.build_step);
@@ -2660,6 +2716,118 @@ void GraphManager::PreRunThread(GraphManager *graph_manager) {
   }
 }
 
+Status GraphManager::DistinguishGetNextAndData(ComputeGraphPtr &graph, vector<NodePtr> &data_nodes,
+                                               vector<NodePtr> &getnext_nosink_nodes,
+                                               vector<NodePtr> &getnext_sink_nodes) {
+  GELOGD("Start distinguish getnext and data node.");
+  for (NodePtr &input_node : graph->GetDirectNode()) {
+    GE_CHECK_NOTNULL(input_node);
+    OpDescPtr op_desc = input_node->GetOpDesc();
+    GE_CHECK_NOTNULL(op_desc);
+    if (op_desc->GetType() == DATA && op_desc->GetName() != kShapeDataName) {
+      if (op_desc->GetName().find(kSubstrOfGetNextNosinkName) == string::npos) {
+        data_nodes.emplace_back(input_node);
+      } else {
+        getnext_nosink_nodes.emplace_back(input_node);
+      }
+    }
+    std::string op_type;
+    auto ret = GetOriginalType(input_node, op_type);
+    if (ret != SUCCESS) {
+      GELOGE(FAILED, "Failed to get node %s original type.", input_node->GetName().c_str());
+      return FAILED;
+    }
+    if (op_type == kGetNextName) {
+      GELOGD("Name of getnext sink is %s.", op_desc->GetName().c_str());
+      getnext_sink_nodes.emplace_back(input_node);
+    }
+  }
+  GELOGI("data count is %zu, getnext nosink count is %zu, getnext sink count is %zu.", data_nodes.size(),
+         getnext_nosink_nodes.size(), getnext_sink_nodes.size());
+  return SUCCESS;
+}
+
+void GraphManager::ParseInputsDimsForData(const std::vector<InputTensorInfo> &input_tensor) {
+  GELOGD("Start parse input dims from data.");
+  for (size_t i = 0; i < input_tensor.size(); ++i) {
+    std::vector<int64_t> dynamic_dim;
+    for (size_t j = 0; j < input_tensor[i].dims.size(); ++j) {
+      dynamic_dim.emplace_back(input_tensor[i].dims[j]);
+    }
+    GELOGD("Input tensor dims is %s.", formats::JoinToString(dynamic_dim).c_str());
+    GetLocalOmgContext().user_real_input_dims.emplace_back(input_tensor[i].dims);
+  }
+}
+
+Status GraphManager::ParseInputsDimsForGetNexNosinkAndData(const vector<NodePtr> &dynamic_nodes,
+                                                           const std::vector<InputTensorInfo> &input_tensor) {
+  GELOGD("Start parse inputs dims when coexist data and getnext sink.");
+  for (size_t i = 0; i < dynamic_nodes.size(); ++i) {
+    auto op_desc = dynamic_nodes.at(i)->GetOpDesc();
+    if (op_desc == nullptr) {
+      continue;
+    }
+    GeAttrValue::INT index = 0;
+    if (!(AttrUtils::GetInt(op_desc, ATTR_NAME_INDEX, index))) {
+      GELOGE(PARAM_INVALID, "Get index from attr failed");
+      return PARAM_INVALID;
+    }
+    if (static_cast<size_t>(index) > input_tensor.size()) {
+      GELOGE(PARAM_INVALID, "The count of input tensor should be equal to the count of data.");
+      return PARAM_INVALID;
+    }
+
+    GetLocalOmgContext().user_real_input_dims.emplace_back(input_tensor.at(index).dims);
+    GELOGI("Shape dims of %d data is %s.", index, formats::JoinToString(input_tensor.at(index).dims).c_str());
+  }
+  return SUCCESS;
+}
+
+Status GraphManager::ParseInputsDims(const std::vector<InputTensorInfo> &input_tensor) {
+  GELOGI("Start parse input dims of %zu input tensor.", input_tensor.size());
+  GetLocalOmgContext().user_real_input_dims.clear();
+  if (!GetLocalOmgContext().dynamic_node_type.empty()) {
+    vector<NodePtr> data_nodes;
+    vector<NodePtr> getnext_nosink_nodes;
+    vector<NodePtr> getnext_sink_nodes;
+    if (DistinguishGetNextAndData(compute_graph_, data_nodes, getnext_nosink_nodes, getnext_sink_nodes) != SUCCESS) {
+      GELOGE(PARAM_INVALID, "Failed to distinguish getnext and data node.");
+      return PARAM_INVALID;
+    }
+    if (GetLocalOmgContext().dynamic_node_type == DATA) {
+      if (getnext_nosink_nodes.empty()) {
+        // just data or data+getnext_sink
+        ParseInputsDimsForData(input_tensor);
+      } else {
+        // data+getnext_nosink, but only need to get shape_dims of data
+        if (ParseInputsDimsForGetNexNosinkAndData(data_nodes, input_tensor) != SUCCESS) {
+          GELOGE(PARAM_INVALID, "Failed to parse dims from data, when data coexist with getnext nosink.");
+          return PARAM_INVALID;
+        }
+      }
+    } else {
+      if (getnext_nosink_nodes.empty()) {
+        // just getnext_sink or getnext_sink+data, need to get shape_dims from aicpu op
+        GELOGI("Need to get dims from aicpu op: GETDYNAMICDIMS.");
+        return SUCCESS;
+      } else {
+        if (data_nodes.empty()) {
+          // just getnext_nosink
+          ParseInputsDimsForData(input_tensor);
+        } else {
+          // getnext_nosink + data, but only need to get shape_dims of getnext_nosink
+          if (ParseInputsDimsForGetNexNosinkAndData(getnext_nosink_nodes, input_tensor) != SUCCESS) {
+            GELOGE(PARAM_INVALID, "Failed to parse dims from getnext nosink, when data coexist with getnext nosink");
+            return PARAM_INVALID;
+          }
+        }
+      }
+    }
+  }
+  GELOGI("Parse %zu inputs dims success.", GetLocalOmgContext().user_real_input_dims.size());
+  return SUCCESS;
+}
+
 void GraphManager::RunThread(GraphManager *graph_manager) {
   if (prctl(PR_SET_NAME, ("GE_Run")) != 0) {
     GELOGW("Set thread name failed.");
@@ -2681,6 +2849,11 @@ void GraphManager::RunThread(GraphManager *graph_manager) {
     if (args.graph_node->graph_run_async_listener_ != nullptr) {
       args.graph_node->graph_run_async_listener_->SetCallback(args.callback);
     }
+    // parse inputs.dims to vector<vector<uint64_t>> dynamic_dims
+    if (graph_manager->ParseInputsDims(args.input_tensor) != SUCCESS) {
+      GELOGE(PARAM_INVALID, "Parse input dims failed.");
+      return;
+    }
 
     Status ret;
     if (!args.graph_node->GetLoadFlag()) {

ge/graph/manager/graph_manager.h

@@ -72,6 +72,7 @@ class GraphManager {
   ///
   Status AddGraph(const GraphId &graph_id, const Graph &graph, const std::map<std::string, std::string> &options,
                   const OmgContext &omg_context);
+  Status InitDynamicParams(ComputeGraphPtr &compute_graph);
 
   ///
   /// @ingroup ge_graph
@@ -215,6 +216,12 @@ class GraphManager {
   static Status ProcessSubGraphWithMultiThreads(GraphManager *graph_manager, GraphId root_graph_id,
                                                 const SubGraphInfoPtr &sub_graph_info_ptr, uint64_t session_id,
                                                 const GEThreadLocalContext &ge_context);
+  Status ParseInputsDims(const std::vector<InputTensorInfo> &input_tensor);
+  Status DistinguishGetNextAndData(ComputeGraphPtr &graph, vector<NodePtr> &data_nodes,
+                                   vector<NodePtr> &getnext_nosink_nodes, vector<NodePtr> &getnext_sink_nodes);
+  void ParseInputsDimsForData(const std::vector<InputTensorInfo> &input_tensor);
+  Status ParseInputsDimsForGetNexNosinkAndData(const vector<NodePtr> &dynamic_nodes,
+                                               const std::vector<InputTensorInfo> &input_tensor);
   Status PreRun(const GraphNodePtr &graph_node, const std::vector<GeTensor> &inputs, GeRootModelPtr &ge_root_model,
                 uint64_t session_id = INVALID_SESSION_ID);
 
@@ -370,7 +377,7 @@ class GraphManager {
   BlockingQueue<RunArgs> run_args_q_{};
   std::thread prerun_thread_;
   std::thread run_thread_;
-
+  ComputeGraphPtr compute_graph_;
   std::map<GraphId, GraphNodePtr> graph_map_;
   std::map<GraphId, ModelCacheHelperPtr> cache_helper_map_;
 

ge/graph/manager/graph_manager_utils.h

@@ -249,6 +249,9 @@ struct GraphManagerOptions {
   std::string save_original_model;
   std::string build_mode;
   std::string build_step;
+  std::string input_shape;
+  std::string dynamic_dims;
+  int32_t dynamic_node_type = -1;
   GraphManagerOptions()
       : stream_num(1),
         perf_level(domi::GEN_TASK_WITHOUT_FUSION),

ge/graph/preprocess/graph_preprocess.cc

@@ -130,6 +130,8 @@ const char *const kMbatchSwitchnName = "mbatch-switch-name";
 
 // the size of user defined output datatype or format string after split by ":".
 const size_t kUserDefinedElementCount = 2;
+const int kDataOutIndex = 0;
+const int64_t kInvalidDynaimcDimsType = -1;
 
 OpDescPtr CreateTensorShape(const GeTensorDesc &data_tensor) {
   GeTensorPtr tensor = MakeShared<GeTensor>();
@@ -1160,6 +1162,9 @@ Status GraphPrepare::UpdateInput(const std::vector<GeTensor> &user_input) {
         return FAILED;
       }
 
+      if (IsDynamicDims(input_node)) {
+        continue;
+      }
       GeTensorDesc desc(user_input[index].GetTensorDesc());
       auto format = desc.GetFormat();
       auto origin_format = desc.GetOriginFormat();
@@ -1221,10 +1226,7 @@ Status GraphPrepare::UpdateInput(const std::vector<GeTensor> &user_input) {
 
       if (!options_.train_graph_flag) {
         Status ret = AdjustDataOpOutput(input_node);
-        if (ret != SUCCESS) {
-          GELOGE(ret, "AdjustDataOpOutput fail, ret:%u", ret);
-          return ret;
-        }
+        GE_IF_BOOL_EXEC(ret != SUCCESS, GELOGE(ret, "AdjustDataOpOutput fail, ret:%u", ret); return ret);
       }
     }
   }
@@ -1570,6 +1572,22 @@ Status GraphPrepare::VerifyConstOp(const NodePtr &node) {
   return SUCCESS;
 }
 
+bool GraphPrepare::IsDynamicDims(const NodePtr &input_node) {
+  auto data_shape = NodeUtils::GetOutputDesc(*input_node, kDataOutIndex).GetShape();
+  const auto &dims = data_shape.GetDims();
+  bool all_is_positive = false;
+  if (std::all_of(dims.begin(), dims.end(), [](int64_t val) { return val >= 0; })) {
+    all_is_positive = true;
+  }
+  if (!all_is_positive && !options_.input_shape.empty() && !options_.dynamic_dims.empty() &&
+      options_.dynamic_node_type != kInvalidDynaimcDimsType) {
+    GELOGI("No need to check and update desc info, the dims of %s is %s.", input_node->GetName().c_str(),
+           formats::JoinToString(dims).c_str());
+    return true;
+  }
+  return false;
+}
+
 Status GraphPrepare::CheckUserInput(const std::vector<GeTensor> &user_input) {
   if (GetLocalOmgContext().is_dynamic_input) {
     return SUCCESS;
@@ -1595,6 +1613,9 @@ Status GraphPrepare::CheckUserInput(const std::vector<GeTensor> &user_input) {
         GELOGE(GE_GRAPH_INIT_FAILED, "user_input size:%zu, data op index:%ld.", user_input.size(), index);
         return GE_GRAPH_INIT_FAILED;
       }
+      if (IsDynamicDims(input_node)) {
+        continue;
+      }
       GeTensorDesc desc(user_input[index].GetTensorDesc());
 
       for (size_t i = 0; i < desc.GetShape().GetDimNum(); ++i) {

ge/graph/preprocess/graph_preprocess.h

@@ -84,6 +84,7 @@ class GraphPrepare {
 
   Status GraphEquivalentTransformation();
   void TypeConversionOfConstant();
+  bool IsDynamicDims(const NodePtr &input_node);
 
   ge::ComputeGraphPtr compute_graph_;
   GraphManagerOptions options_;

ge/graph/preprocess/multi_batch_copy_graph.h

@@ -55,9 +55,7 @@ class MultiBatchGraphCopyer {
       data_name_order_.push_back(item.first);
     }
   }
-  void SetDataToDynamicInfo(const map<string, vector<vector<int64_t>>> &designate_shape) {
-    data_to_dynamic_info_ = designate_shape;
-  }
+
   void SetDynamicType(const DynamicType dynamic_type) {
     dynamic_type_ = dynamic_type;
   }
@@ -69,15 +67,26 @@ class MultiBatchGraphCopyer {
 
   // label status for origin_all_nodes_
   Status LabelStatus();
+  Status LabelInBatchBranchStatus();
+  void LabelStatusForData(const NodePtr &data);
+  void LabelStatusForGetNextSink(const NodePtr &data);
   // add nodes functions
   Status CreateNewNodes();
 
   NodePtr InsertShapeDataNode();
-  Status InsertSwitchNForData(const NodePtr &data);
+  NodePtr InsertGetDynamicDimsNode();
+
+  Status InsertSwitchNAndUpdateMaxShape(const NodePtr &node);
+
+  Status InsertSwitchNForData(const NodePtr &node, const size_t &out_anchor_index, const size_t &peer_in_anchor_index,
+                              std::vector<std::pair<Node *, NodePtr>> &dynamic_out_to_switchn);
+
   Status InsertIdentityAfterSwitchN();
-  Status UpdateMaxShapeToData(const NodePtr &data);
+  Status UpdateMaxShapeToData(const NodePtr &node, size_t out_anchor_index);
+  Status UpdateShapeOfShapeNode(const NodePtr &node, size_t out_anchor_index);
 
   Status InsertMergeForEdgeNode(const NodePtr &node);
+  Status LinkGetDynamicDimsToNetOutput(const NodePtr &node);
 
   /// Insert a merge node for src node `node` on output index `index`. The merge node will be used to merge all nodes
   /// in batch-branch to one output to the node out of the batch-branch.
@@ -95,12 +104,16 @@ class MultiBatchGraphCopyer {
 
   // link edges functions
   Status LinkEdges();
-  Status LinkDataToSwitchN(const NodePtr &data);
+  Status AddAttrForGetDynamicDims(const NodePtr &node);
+  Status AddLinkForGetDynamicDims(const NodePtr &node);
+
+  Status LinkDataToSwitchN(const NodePtr &data, const NodePtr &switchn, const int &out_index);
   Status LinkToMerge(const NodePtr &node);
   Status LinkToNodeInBranch(const NodePtr &node);
   Status LinkToNodeOutBranch(const NodePtr &node);
-  Status LinkDataToMerge(const NodePtr &data, const NodePtr &merge);
+  Status LinkDataToMerge(const NodePtr &data, const NodePtr &merge, const NodePtr &switchn);
   Status LinkNodeToMerge(const NodePtr &node, int out_index, const NodePtr &merge);
+  NodePtr FindSwitchnNodeForDataEdge(const OutDataAnchorPtr &data_out_anchor, const NodePtr &origin_node);
   Status CopyInDataEdges(const NodePtr &origin_node, int batch_num, const NodePtr &copyed_node);
   Status CopyInControlEdges(const NodePtr &node, int batch_num, const NodePtr &copyed_node);
   Status CheckAndParseDynamicData();
@@ -127,6 +140,11 @@ class MultiBatchGraphCopyer {
   // the data nodes, and the SwitchN nodes inserted after it
   std::map<Node *, NodePtr> data_nodes_to_switchn_;
 
+  // the getnext_sink nodes, and the SwitchN nodes inserted after it
+  std::vector<std::vector<std::pair<Node *, NodePtr>>> getnext_nodes_to_switchn_;
+  std::vector<std::vector<std::pair<int, int>>> outidx_inidx_mappings_;
+  std::vector<std::pair<int, int>> outidx_inidx_mapping_;
+
   // the nodes on the in/out-batch-branch edge, and the merge nodes inserted after it
   std::map<Node *, std::vector<NodePtr>> nodes_to_merge_nodes_;
 
@@ -142,6 +160,9 @@ class MultiBatchGraphCopyer {
 
   // dynamic type : dynamic batch,, dynamic image size, dynamic dims.
   DynamicType dynamic_type_ = DynamicType::kDynamicUnknown;
+
+  std::vector<std::pair<size_t, size_t>> getnext_sink_dynamic_out_mapping_;
+  bool getnext_sink_dynamic_dims_ = false;
 };
 }  // namespace multibatch
 }  // namespace ge

ge/graph/preprocess/multi_batch_options.cc

@@ -27,6 +27,9 @@
 #include "graph/ge_context.h"
 #include "graph/common/local_context.h"
 #include "framework/common/types.h"
+#include "graph/compute_graph.h"
+#include "graph/utils/graph_utils.h"
+#include "graph/common/omg_util.h"
 
 namespace ge {
 namespace multibatch {
@@ -38,6 +41,18 @@ const int kDynamicBatchDynamicDimsNum = 1;
 const int kDynamicImgSizeDynamciDimsNum = 2;
 const size_t kMaxNDDimNum = 4;
 const size_t kMinNDDimNum = 1;
+const size_t kNumOfGetnextNode = 1;
+const int kDivisionConst = 2;
+const char *const kSubstrOfGetNextNosinkName = "IteratorGetNext";
+const char *const kShapeDataName = "ascend_mbatch_shape_data";
+const char *const kGetNextName = "IteratorV2";
+
+inline bool IsGetNextType(const NodePtr &node) {
+  std::string original_type;
+  GE_IF_BOOL_EXEC(GetOriginalType(node, original_type) != SUCCESS,
+                  GELOGW("Get original type failed."); return false);
+  return (original_type == kGetNextName);
+}
 
 void ParseDynamicSize(string dynamic_size, vector<vector<int64_t>> &shapes) {
   std::vector<std::string> shape_strs = ge::StringUtils::Split(dynamic_size, ';');
@@ -59,6 +74,256 @@ void ParseDynamicSize(string dynamic_size, vector<vector<int64_t>> &shapes) {
   }
 }
 
+Status DistinguishGetNextAndData(ComputeGraphPtr &graph, vector<NodePtr> &data_nodes,
+                                 vector<NodePtr> &getnext_nosink_nodes, vector<NodePtr> &getnext_sink_nodes) {
+  GELOGD("Start distinguish getnext and data node.");
+  for (NodePtr &input_node : graph->GetDirectNode()) {
+    GE_CHECK_NOTNULL(input_node);
+    OpDescPtr op_desc = input_node->GetOpDesc();
+    GE_CHECK_NOTNULL(op_desc);
+    if (op_desc->GetType() == DATA && op_desc->GetName() != kShapeDataName) {
+      if (op_desc->GetName().find(kSubstrOfGetNextNosinkName) == string::npos) {
+        data_nodes.emplace_back(input_node);
+      } else {
+        getnext_nosink_nodes.emplace_back(input_node);
+      }
+    }
+    if (IsGetNextType(input_node)) {
+      GELOGD("Name of getnext sink is %s.", op_desc->GetName().c_str());
+      getnext_sink_nodes.emplace_back(input_node);
+    }
+  }
+  GELOGI("Data count is %zu, getnext nosink count is %zu, getnext sink count is %zu.", data_nodes.size(),
+         getnext_nosink_nodes.size(), getnext_sink_nodes.size());
+  return SUCCESS;
+}
+
+Status CheckSequenceOfData(ComputeGraphPtr &graph, const vector<NodePtr> &data_nodes) {
+  GELOGD("Start check input sequence from data nodes and input shape.");
+  if (data_nodes.size() != GetLocalOmgContext().user_input_dims.size()) {
+    GELOGE(PARAM_INVALID, "The count of input shape:%zu should be equal to the count of data num:%zu.",
+           GetLocalOmgContext().user_input_dims.size(), data_nodes.size());
+    return PARAM_INVALID;
+  }
+  for (size_t i = 0; i < data_nodes.size(); ++i) {
+    auto data_node = data_nodes.at(i);
+    GE_CHECK_NOTNULL(data_node);
+    GE_CHECK_NOTNULL(data_node->GetOpDesc());
+    auto output_shape = data_node->GetOpDesc()->GetOutputDesc(0).GetShape().GetDims();
+    auto dynamic_dims = GetLocalOmgContext().user_input_dims.at(i).second;
+    if (output_shape.empty() && dynamic_dims.size() == 1 && dynamic_dims.at(0) == 0) {
+      GELOGI("No need to check sequence for constant.");
+      continue;
+    }
+    if (dynamic_dims.size() != output_shape.size()) {
+      GELOGE(PARAM_INVALID, "The output shape of %s is %s, the input shape from options of %s is %s.",
+             data_node->GetName().c_str(), formats::JoinToString(output_shape).c_str(),
+             GetLocalOmgContext().user_input_dims.at(i).first.c_str(), formats::JoinToString(dynamic_dims).c_str());
+      return PARAM_INVALID;
+    }
+    for (size_t j = 0; j < dynamic_dims.size(); ++j) {
+      if (dynamic_dims.at(j) != kDynmaicDims && dynamic_dims.at(j) != output_shape.at(j)) {
+        GELOGE(INTERNAL_ERROR, "Value of input shape %s should be equal to %s.",
+               formats::JoinToString(dynamic_dims).c_str(), formats::JoinToString(output_shape).c_str());
+        return INTERNAL_ERROR;
+      }
+    }
+  }
+  return SUCCESS;
+}
+
+Status CheckSequenceOfGetnext(ComputeGraphPtr &graph, const vector<NodePtr> &getnext_sink_node) {
+  GELOGD("Start check input sequence from getnext sink nodes and input shape.");
+  if (getnext_sink_node.size() != kNumOfGetnextNode) {
+    GELOGE(PARAM_INVALID, "Not support dynamic dims when a graph with multi getnext nodes.");
+    return PARAM_INVALID;
+  }
+  auto data_node = getnext_sink_node.at(0);
+  GE_CHECK_NOTNULL(data_node);
+  auto op_desc = data_node->GetOpDesc();
+  GE_CHECK_NOTNULL(op_desc);
+  size_t data_count = data_node->GetAllOutDataAnchors().size() / kDivisionConst;
+  if (data_count != GetLocalOmgContext().user_input_dims.size()) {
+    GELOGE(PARAM_INVALID, "Output count of %s is %zu, should be equal to count of input shape: %zu",
+           op_desc->GetName().c_str(), data_count, GetLocalOmgContext().user_input_dims.size());
+    return PARAM_INVALID;
+  }
+  for (size_t i = 0; i < data_count; ++i) {
+    auto output_shape = data_node->GetOpDesc()->GetOutputDesc(i).GetShape().GetDims();
+    auto dynamic_dims = GetLocalOmgContext().user_input_dims.at(i).second;
+    if (output_shape.empty() && dynamic_dims.size() == 1 && dynamic_dims.at(0) == 0) {
+      GELOGI("No need to check sequence for constant.");
+      continue;
+    }
+    if (dynamic_dims.size() != output_shape.size()) {
+      GELOGE(PARAM_INVALID, "the output_shape of %s is %s, the input_shape from options of %s is %s.",
+             data_node->GetName().c_str(), formats::JoinToString(output_shape).c_str(),
+             GetLocalOmgContext().user_input_dims.at(i).first.c_str(), formats::JoinToString(dynamic_dims).c_str());
+      return PARAM_INVALID;
+    }
+    for (size_t j = 0; j < dynamic_dims.size(); ++j) {
+      if (dynamic_dims.at(j) != kDynmaicDims && dynamic_dims.at(j) != output_shape.at(j)) {
+        GELOGE(INTERNAL_ERROR, "value of input_shape %s should be equal to %s.",
+               formats::JoinToString(dynamic_dims).c_str(), formats::JoinToString(output_shape).c_str());
+        return INTERNAL_ERROR;
+      }
+    }
+  }
+  return SUCCESS;
+}
+
+Status CheckSequenceOfOptions(ComputeGraphPtr &graph, vector<NodePtr> &data_nodes,
+                              vector<NodePtr> &getnext_nosink_nodes, vector<NodePtr> &getnext_sink_nodes) {
+  if (GetLocalOmgContext().dynamic_node_type.empty()) {
+    GELOGI("No need to CheckSequenceOfOptions.");
+    return SUCCESS;
+  }
+
+  if (DistinguishGetNextAndData(graph, data_nodes, getnext_nosink_nodes, getnext_sink_nodes) != SUCCESS) {
+    GELOGE(PARAM_INVALID, "DistinguishGetNextAndData failed.");
+    return PARAM_INVALID;
+  }
+
+  if (GetLocalOmgContext().dynamic_node_type == DATA) {
+    GELOGD("Users want data nodes to be dynamic.");
+    if (CheckSequenceOfData(graph, data_nodes) != SUCCESS) {
+      GELOGE(PARAM_INVALID, "Failed to check sequence of data nodes.");
+      return PARAM_INVALID;
+    }
+  } else {
+    GELOGD("Users want getnext nodes to be dynamic.");
+    if (!getnext_nosink_nodes.empty()) {
+      if (CheckSequenceOfData(graph, getnext_nosink_nodes) != SUCCESS) {
+        GELOGE(PARAM_INVALID, "Failed to check sequence of getnext nosink nodes.");
+        return PARAM_INVALID;
+      }
+    } else {
+      if (CheckSequenceOfGetnext(graph, getnext_sink_nodes) != SUCCESS) {
+        GELOGE(PARAM_INVALID, "Failed to check sequence of getnext sink nodes.");
+        return PARAM_INVALID;
+      }
+    }
+  }
+  return SUCCESS;
+}
+
+Status UpdateNameOfData(ComputeGraphPtr &graph, const vector<NodePtr> &data_nodes) {
+  GELOGD("Update first value of input shape by data nodes.");
+  if (data_nodes.size() != GetLocalOmgContext().user_input_dims.size()) {
+    GELOGE(PARAM_INVALID, "count of data_nodes: %zu should be equal to input_shape count: %zu.",
+           data_nodes.size(), GetLocalOmgContext().user_input_dims.size());
+    return PARAM_INVALID;
+  }
+  for (size_t i = 0; i < data_nodes.size(); ++i) {
+    GELOGD("The %zu data name is %s.", i, data_nodes.at(i)->GetOpDesc()->GetName().c_str());
+    GetLocalOmgContext().user_input_dims.at(i).first = data_nodes.at(i)->GetOpDesc()->GetName();
+  }
+  return SUCCESS;
+}
+
+Status UpdateNameOfGetnext(ComputeGraphPtr &graph, const vector<NodePtr> &getnext_sink_nodes) {
+  GELOGD("Update first value of input shape by getnext sink nodes.");
+  if (getnext_sink_nodes.size() != kNumOfGetnextNode) {
+    GELOGE(PARAM_INVALID, "Not support dynamic dims when a graph with multi getnext nodes.");
+    return PARAM_INVALID;
+  }
+  auto input_node = getnext_sink_nodes.at(0);
+  GE_CHECK_NOTNULL(input_node);
+  auto op_desc = input_node->GetOpDesc();
+  GE_CHECK_NOTNULL(op_desc);
+  // user want getnext dynamic, just getnext or data+getnext_sink
+  size_t data_count = input_node->GetAllOutDataAnchors().size() / kDivisionConst;
+  if (data_count != GetLocalOmgContext().user_input_dims.size()) {
+    GELOGE(PARAM_INVALID, "Output count of %s is %zu, should be equal to count of input shape: %zu",
+           op_desc->GetName().c_str(), data_count, GetLocalOmgContext().user_input_dims.size());
+    return PARAM_INVALID;
+  }
+
+  for (size_t i = 0; i < data_count; ++i) {
+    string data_name = op_desc->GetName() + "_" + std::to_string(i);
+    GELOGD("Data just from getnext sink is %s.", data_name.c_str());
+    GetLocalOmgContext().user_input_dims.at(i).first = data_name;
+  }
+  return SUCCESS;
+}
+
+// need to distinguish online and offline, offline no need to update the name of input_shape
+Status UpdateNameOfInputShape(ComputeGraphPtr &graph, const vector<NodePtr> &data_nodes,
+                              const vector<NodePtr> &getnext_nosink_nodes, const vector<NodePtr> &getnext_sink_nodes) {
+  if (GetLocalOmgContext().dynamic_node_type.empty()) {
+    GELOGI("No need to update first value of input shape when offline infer.");
+    return SUCCESS;
+  }
+
+  if (GetLocalOmgContext().dynamic_node_type == DATA) {
+    GELOGD("Users want data nodes to be dynamic.");
+    if (UpdateNameOfData(graph, data_nodes) != SUCCESS) {
+      GELOGE(PARAM_INVALID, "Failed to update first value of input shape of data nodes.");
+      return PARAM_INVALID;
+    }
+  } else {
+    GELOGD("Users want getnext nodes to be dynamic.");
+    if (!getnext_nosink_nodes.empty()) {
+      if (UpdateNameOfData(graph, getnext_nosink_nodes) != SUCCESS) {
+        GELOGE(PARAM_INVALID, "Failed to update first value of input shape of getnext nosink nodes.");
+        return PARAM_INVALID;
+      }
+    } else {
+      if (UpdateNameOfGetnext(graph, getnext_sink_nodes) != SUCCESS) {
+        GELOGE(PARAM_INVALID, "Failed to update first value of input shape of getnext sink nodes.");
+        return PARAM_INVALID;
+      }
+    }
+  }
+  return SUCCESS;
+}
+
+Status DeleteIdentityInsertByAdapter(ComputeGraphPtr &graph) {
+  GELOGD("Start delete identity node inserted by adapter.");
+  for (NodePtr &node : graph->GetDirectNode()) {
+    GE_CHECK_NOTNULL(node);
+    OpDescPtr op_desc = node->GetOpDesc();
+    GE_CHECK_NOTNULL(op_desc);
+    if (IsGetNextType(node)) {
+      for (auto &out_data_anchor : node->GetAllOutDataAnchors()) {
+        GE_IF_BOOL_EXEC(out_data_anchor == nullptr, continue);
+        for (auto &peer_in_anchor : out_data_anchor->GetPeerInDataAnchors()) {
+          GE_IF_BOOL_EXEC(peer_in_anchor == nullptr, continue);
+          auto dst_node = peer_in_anchor->GetOwnerNode();
+          GE_IF_BOOL_EXEC(dst_node == nullptr, continue);
+          if (dst_node->GetType() == IDENTITY) {
+            GELOGI("Need to remove %s.", dst_node->GetName().c_str());
+            if (ge::GraphUtils::RemoveNodeWithoutRelink(graph, dst_node) != GRAPH_SUCCESS) {
+              GELOGE(FAILED, "Remove Identity node %s failed.", dst_node->GetName().c_str());
+              return FAILED;
+            }
+          }
+        }
+      }
+    }
+  }
+  return SUCCESS;
+}
+
+Status CheckNegativeCountOfOptions(const std::vector<std::vector<int64_t>> &shapes) {
+  size_t negative_count = 0;
+  for (size_t i = 0; i < GetLocalOmgContext().user_input_dims.size(); ++i) {
+    for (size_t j = 0; j < GetLocalOmgContext().user_input_dims.at(i).second.size(); ++j) {
+      if (GetLocalOmgContext().user_input_dims.at(i).second.at(j) == kDynmaicDims) {
+        negative_count++;
+      }
+    }
+  }
+  for (size_t i = 0; i < shapes.size(); ++i) {
+    if (shapes.at(i).size() != negative_count) {
+      GELOGE(PARAM_INVALID, "Each gear num of dynamic_dims is %zu should be equal to %zu.", shapes.at(i).size(),
+             negative_count);
+      return PARAM_INVALID;
+    }
+  }
+  return SUCCESS;
+}
+
 ///
 /// @ingroup ge
 /// @brief Init Dynamic Param from Options.
@@ -115,8 +380,10 @@ Status ParserDataToDynmaicInfo(const vector<vector<int64_t>> &shapes,
     auto &data_shape = cur_item.second;
     auto dynamic_dims_num = std::count_if(data_shape.begin(), data_shape.end(),
                                           [&data_shape](int64_t dim){ return dim < 0; });
+    GELOGI("Train_Dynamic dynamic_dims_num of %s is %zu", data_name.c_str(), dynamic_dims_num);
     vector<vector<int64_t> > dynamic_info;
     for (auto &dynamic_gear_info : shapes) {
+      GELOGI("Train_Dynamic dynamic_gear_info is %s", formats::JoinToString(dynamic_gear_info).c_str());
       vector<int64_t> one_gear;
       if (dynamic_gear_info.size() == static_cast<size_t>(dynamic_dims_num)) {
         one_gear = dynamic_gear_info;
@@ -139,6 +406,7 @@ Status ParserDataToDynmaicInfo(const vector<vector<int64_t>> &shapes,
                data_name.c_str(), formats::JoinToString(data_shape).c_str());
         return FAILED;
       }
+      GELOGI("Train_Dynamic one_gear is %s.", formats::JoinToString(one_gear).c_str());
       dynamic_info.push_back(one_gear);
     }
     cur_data_index += dynamic_dims_num;
@@ -218,7 +486,7 @@ Status CalcShape(const std::vector<int64_t> &batch_shape, GeShape &data_shape) {
         ErrorManager::GetInstance().ATCReportErrMessage(
             "E19012", {"function", "reason"},
             {"CalcShape", "the batch shape count " + std::to_string(batch_shape.size()) +
-                              " does not match the data shape " + data_shape.ToString()});
+                " does not match the data shape " + data_shape.ToString()});
         GELOGE(PARAM_INVALID,
                "Failed to calc tensor shape, the batch shape count %zu, does not match the data shape %s",
                batch_shape.size(), data_shape.ToString().c_str());
@@ -227,6 +495,7 @@ Status CalcShape(const std::vector<int64_t> &batch_shape, GeShape &data_shape) {
       data_shape.SetDim(i, batch_shape[batch_shape_index++]);
     }
   }
+  GELOGI("CalcShape size of batch_shape is %zu, batch_shape_index is %zu.", batch_shape.size(), batch_shape_index);
   if (batch_shape_index != batch_shape.size()) {
     ErrorManager::GetInstance().ATCReportErrMessage(
         "E19012", {"function", "reason"}, {"CalcShape", "the batch shape count " + std::to_string(batch_shape.size()) +

ge/graph/preprocess/multi_batch_options.h

@@ -28,6 +28,21 @@ namespace ge {
 namespace multibatch {
 ///
 /// @ingroup ge
+/// @brief Update Dynamic Param from Options.
+/// @param [in] ComputeGraphPtr &graph: the train graph
+/// @return SUCCESS: valid / PARAM_INVALID: invalid.
+///
+Status CheckSequenceOfOptions(ComputeGraphPtr &graph, std::vector<NodePtr> &data_nodes,
+                              std::vector<NodePtr> &getnext_nosink_nodes, std::vector<NodePtr> &getnext_sink_nodes);
+
+Status UpdateNameOfInputShape(ComputeGraphPtr &graph, const vector<NodePtr> &data_nodes,
+                              const vector<NodePtr> &getnext_nosink_nodes, const vector<NodePtr> &getnext_sink_nodes);
+
+Status DeleteIdentityInsertByAdapter(ComputeGraphPtr &graph);
+
+Status CheckNegativeCountOfOptions(const std::vector<std::vector<int64_t>> &shapes);
+///
+/// @ingroup ge
 /// @brief Init Dynamic Param from Options.
 /// @param [out] std::vector<std::vector<int64_t>> &shapes: Result for Params.
 /// @return true: Configed for Multi batch / false: Not configed for Multi batch.

inc/external/ge/ge_api_types.h

@@ -174,6 +174,9 @@ const std::string HCOM_PARALLEL = "ge.hcomParallel";
 // configure whether to use dynamic batch size
 const char *const kDynamicBatchSize = "ge.dynamicBatchSize";
 
+const std::string INPUT_SHAPE = "ge.inputShape";
+
+const std::string DYNAMIC_NODE_TYPE = "ge.dynamicNodeType";
 // configure whether to use dynamic image size
 const char *const kDynamicImageSize = "ge.dynamicImageSize";
 

inc/framework/common/types.h

@@ -526,6 +526,9 @@ REGISTER_OPTYPE_DECLARE(HVDCALLBACKALLGATHER, "HorovodAllgather");
 REGISTER_OPTYPE_DECLARE(HVDCALLBACKBROADCAST, "HorovodBroadcast");
 REGISTER_OPTYPE_DECLARE(HVDWAIT, "HorovodWait");
 
+// aicpu op for online_infer dynamic_dims
+REGISTER_OPTYPE_DECLARE(GETDYNAMICDIMS, "GetDynamicDims");
+
 enum InputMode { INPUT = 0, CONST };
 
 // Definition of the processing status enum of the process module

inc/framework/omg/omg_inner_types.h

@@ -115,6 +115,10 @@ struct OmgContext {
   std::string dynamic_batch_size;
   std::string dynamic_image_size;
   std::string dynamic_dims;
+  std::string dynamic_node_type;
+  std::vector<std::vector<int64_t>> user_real_input_dims;
+  std::vector<int64_t> cur_dynamic_dims;
+  bool need_multi_batch = false;
 };
 }  // namespace ge
 

metadef

@@ -1 +1 @@
-Subproject commit 37465b85d30b67a0edcc6ea4acd2f11a9697c7af
+Subproject commit 2f299e7b532f0cc7031c35e5f0fe7a52ee4f4697

parser

@@ -1 +1 @@
-Subproject commit 5fa1f3ed9b1785b9fd1623d624de91838dff615e
+Subproject commit 40c6b462724ad0bc1b76df1fb4fbd0df0e6c0536