From cb397142ac88fa7aa7b7994696945cbfb705afba Mon Sep 17 00:00:00 2001 From: lichun Date: Mon, 23 Nov 2020 20:24:13 +0800 Subject: [PATCH] Feature:infer support dynamic shape --- ge/executor/ge_executor.cc | 1 + .../passes/mark_graph_unknown_status_pass.cc | 9 ++++++++ .../executor/hybrid_model_async_executor.cc | 8 +++++-- ge/hybrid/executor/rt_callback_manager.cc | 2 +- ge/hybrid/executor/subgraph_executor.cc | 2 +- .../executor/worker/shape_inference_engine.cc | 2 +- ge/hybrid/model/hybrid_model_builder.cc | 18 ++++++++++++---- .../aicore/aicore_node_executor.cc | 3 +++ ge/hybrid/node_executor/node_executor.cc | 21 +++++++++++-------- inc/framework/common/ge_types.h | 21 +++++-------------- metadef | 2 +- parser | 2 +- 12 files changed, 55 insertions(+), 36 deletions(-) diff --git a/ge/executor/ge_executor.cc b/ge/executor/ge_executor.cc index c7836174..86b4de54 100755 --- a/ge/executor/ge_executor.cc +++ b/ge/executor/ge_executor.cc @@ -66,6 +66,7 @@ void GetGeTensorDescFromDomiInfo(std::vector &ge_descs, ge::Shape ge_shape(shape_dims); ge_desc.SetShape(ge_shape); ge_desc.SetSize(desc_item.size); + ge_desc.SetShapeRange(desc_item.shape_info.shape_ranges); ge_descs.emplace_back(ge_desc); ++idx; } diff --git a/ge/graph/passes/mark_graph_unknown_status_pass.cc b/ge/graph/passes/mark_graph_unknown_status_pass.cc index d8f5feff..ae0919fe 100644 --- a/ge/graph/passes/mark_graph_unknown_status_pass.cc +++ b/ge/graph/passes/mark_graph_unknown_status_pass.cc @@ -19,6 +19,10 @@ #include "graph/debug/ge_attr_define.h" namespace ge { +namespace { +const char *const kOwnerGraphIsUnknown = "OwnerGraphIsUnknown"; +} + Status MarkGraphUnknownStatusPass::Run(ComputeGraphPtr graph) { GE_CHECK_NOTNULL(graph); bool is_unknown_shape = false; @@ -35,6 +39,11 @@ Status MarkGraphUnknownStatusPass::Run(ComputeGraphPtr graph) { break; } } + + for (const auto &node : graph->GetDirectNode()) { + GELOGD("Set OwnerGraphIsUnknown attr to node[%s]", node->GetName().c_str()); + (void)AttrUtils::SetBool(node->GetOpDesc(), kOwnerGraphIsUnknown, is_unknown_shape); + } graph->SetGraphUnknownFlag(is_unknown_shape); GELOGD("mark graph [%s] unknown status success! value is %d", graph->GetName().c_str(), is_unknown_shape); return SUCCESS; diff --git a/ge/hybrid/executor/hybrid_model_async_executor.cc b/ge/hybrid/executor/hybrid_model_async_executor.cc index d4652a91..1f738647 100644 --- a/ge/hybrid/executor/hybrid_model_async_executor.cc +++ b/ge/hybrid/executor/hybrid_model_async_executor.cc @@ -58,7 +58,7 @@ Status HybridModelAsyncExecutor::Start(const std::shared_ptr &lis run_flag_ = true; listener_ = listener; - future_ = std::async([&]() -> Status { + future_ = std::async(std::launch::async, [&]() -> Status { GetContext().SetSessionId(executor_->GetContext()->session_id); return RunInternal(); }); @@ -72,7 +72,11 @@ Status HybridModelAsyncExecutor::Stop() { std::lock_guard lk(mu_); run_flag_ = false; data_inputer_->Stop(); - auto ret = future_.get(); + + Status ret = SUCCESS; + if (future_.valid()) { + ret = future_.get(); + } if (stream_ != nullptr) { GE_CHK_RT(rtStreamDestroy(stream_)); diff --git a/ge/hybrid/executor/rt_callback_manager.cc b/ge/hybrid/executor/rt_callback_manager.cc index 63eb46d5..0523a6a5 100644 --- a/ge/hybrid/executor/rt_callback_manager.cc +++ b/ge/hybrid/executor/rt_callback_manager.cc @@ -49,7 +49,7 @@ Status CallbackManager::RegisterCallback(rtCallback_t callback, void *user_data) Status CallbackManager::Init() { rtContext_t ctx = nullptr; GE_CHK_RT_RET(rtCtxGetCurrent(&ctx)); - ret_future_ = std::async([&](rtContext_t context) ->Status { + ret_future_ = std::async(std::launch::async, [&](rtContext_t context) ->Status { return CallbackProcess(context); }, ctx); if (!ret_future_.valid()) { diff --git a/ge/hybrid/executor/subgraph_executor.cc b/ge/hybrid/executor/subgraph_executor.cc index 573e405e..76a6cc37 100644 --- a/ge/hybrid/executor/subgraph_executor.cc +++ b/ge/hybrid/executor/subgraph_executor.cc @@ -307,7 +307,7 @@ Status SubgraphExecutor::LaunchTasks() { Status SubgraphExecutor::ScheduleTasks() { GELOGD("[%s] Start to schedule prepare workers.", graph_item_->GetName().c_str()); - auto prepare_future = std::async([&]() -> Status { + auto prepare_future = std::async(std::launch::async, [&]() -> Status { GetContext().SetSessionId(context_->session_id); auto ret = PrepareNodes(); ready_queue_.Push(nullptr); diff --git a/ge/hybrid/executor/worker/shape_inference_engine.cc b/ge/hybrid/executor/worker/shape_inference_engine.cc index f4dec60a..efa4230f 100755 --- a/ge/hybrid/executor/worker/shape_inference_engine.cc +++ b/ge/hybrid/executor/worker/shape_inference_engine.cc @@ -62,7 +62,7 @@ Status ShapeInferenceEngine::InferShape(NodeState &node_state) { { std::lock_guard lk(mu_); RECORD_SHAPE_INFERENCE_EVENT(execution_context_, node_item.NodeName().c_str(), "[InferShapeAndType] Start"); - GE_CHK_STATUS_RET(ShapeRefiner::InferShapeAndType(node_item.node), "Invoke InferShapeAndType failed."); + GE_CHK_STATUS_RET(ShapeRefiner::InferShapeAndTypeForRunning(node_item.node, true), "Invoke InferShapeAndType failed."); RECORD_SHAPE_INFERENCE_EVENT(execution_context_, node_item.NodeName().c_str(), "[InferShapeAndType] End"); } // Check again to make sure shape is valid after shape inference diff --git a/ge/hybrid/model/hybrid_model_builder.cc b/ge/hybrid/model/hybrid_model_builder.cc index e484d973..2be38897 100755 --- a/ge/hybrid/model/hybrid_model_builder.cc +++ b/ge/hybrid/model/hybrid_model_builder.cc @@ -37,6 +37,16 @@ const uint32_t kSubgraphIndex = 0U; const uint32_t kVarOutputIndex = 0U; const uint32_t kAlignment = 32; const int kBytes = 8; +const char *const kOwnerGraphIsUnknown = "OwnerGraphIsUnknown"; + +bool IsGraphUnknown(ComputeGraph &graph) { + for (const auto &node : graph.GetDirectNode()) { + bool is_unknown_shape = false; + (void)AttrUtils::GetBool(node->GetOpDesc(), kOwnerGraphIsUnknown, is_unknown_shape); + return is_unknown_shape; + } + return false; +} int64_t CalcVarSizeInBytes(const GeTensorDesc &desc) { int64_t var_size = 0; @@ -556,7 +566,7 @@ Status HybridModelBuilder::UnfoldSubgraphs(ComputeGraph &root_graph, ComputeGrap auto subgraph = NodeUtils::GetSubgraph(*node, kSubgraphIndex); GE_CHECK_NOTNULL(subgraph); - bool is_unknown_shape = subgraph->GetGraphUnknownFlag(); + bool is_unknown_shape = IsGraphUnknown(*subgraph); if (!is_unknown_shape) { merged_graph->AddNode(node); GELOGD("[%s] Known shape partitioned call added to merged graph.", op_desc->GetName().c_str()); @@ -603,7 +613,7 @@ Status HybridModelBuilder::UnfoldSubgraph(ComputeGraph &root_graph, if (sub_op_type == PARTITIONEDCALL) { auto sub_sub_graph = NodeUtils::GetSubgraph(*sub_node, kSubgraphIndex); GE_CHECK_NOTNULL(sub_sub_graph); - if (sub_sub_graph->GetGraphUnknownFlag()) { + if (IsGraphUnknown(*sub_sub_graph)) { GE_CHK_STATUS_RET(UnfoldSubgraph(root_graph, parent_graph, *sub_sub_graph), "[%s] Failed to merge subgraph", sub_sub_graph->GetName().c_str()); @@ -693,7 +703,7 @@ Status HybridModelBuilder::LoadGraph() { continue; } - if (sub_graph->GetGraphUnknownFlag()) { + if (IsGraphUnknown(*sub_graph)) { GE_CHK_STATUS_RET(LoadDynamicSubgraph(*sub_graph, false), "Failed to load subgraph: [%s]", sub_graph->GetName().c_str()); @@ -947,7 +957,7 @@ Status HybridModelBuilder::IndexTaskDefs() { continue; } - bool is_unknown_shape = sub_graph->GetGraphUnknownFlag(); + bool is_unknown_shape = IsGraphUnknown(*sub_graph); if (!is_unknown_shape) { GE_CHK_STATUS_RET_NOLOG(LoadGeModel(*sub_graph, ge_model)); continue; diff --git a/ge/hybrid/node_executor/aicore/aicore_node_executor.cc b/ge/hybrid/node_executor/aicore/aicore_node_executor.cc index dbd784c6..2efd8111 100755 --- a/ge/hybrid/node_executor/aicore/aicore_node_executor.cc +++ b/ge/hybrid/node_executor/aicore/aicore_node_executor.cc @@ -244,6 +244,9 @@ void TaskCompilerFactory::Register(CreateFn fn) { } std::unique_ptr TaskCompilerFactory::GetTaskCompiler() { + if (compiler_func_ == nullptr) { + return nullptr; + } auto compiler_instance = std::unique_ptr(compiler_func_()); return compiler_instance; } diff --git a/ge/hybrid/node_executor/node_executor.cc b/ge/hybrid/node_executor/node_executor.cc index 7f11c20a..4055ec4d 100755 --- a/ge/hybrid/node_executor/node_executor.cc +++ b/ge/hybrid/node_executor/node_executor.cc @@ -18,6 +18,7 @@ #include "framework/common/debug/log.h" #include "graph/utils/node_utils.h" #include "init/gelib.h" +#include "graph/utils/tensor_utils.h" #include "hybrid/model/hybrid_model.h" #include "graph/debug/ge_attr_define.h" #include "opskernel_manager/ops_kernel_builder_manager.h" @@ -32,6 +33,7 @@ const char *const kEngineNameAiCpuTf = "aicpu_tf_kernel"; const char *const kEngineNameHccl = "ops_kernel_info_hccl"; const char *const kEngineNameRts = "DNN_VM_RTS_OP_STORE"; const char *const kEngineNameHostCpu = "DNN_VM_HOST_CPU_OP_STORE"; +const char *const kOwnerGraphIsUnknown = "OwnerGraphIsUnknown"; } Status NodeExecutor::PrepareTask(NodeTask &task, TaskContext &context) const { GE_CHK_STATUS_RET_NOLOG(context.AllocateOutputs()); @@ -80,16 +82,17 @@ NodeExecutorManager::ExecutorType NodeExecutorManager::ResolveExecutorType(Node auto op_type = node.GetType(); if (op_type == PARTITIONEDCALL) { const auto &subgraph = NodeUtils::GetSubgraph(node, 0); - if (subgraph != nullptr && subgraph->GetGraphUnknownFlag()) { - GELOGD("node %s was marked as unknown shape in node_executor.", node.GetName().c_str()); - return ExecutorType::DYNAMIC_SUBGRAPH; - } - bool is_dynamic = false; - (void) NodeUtils::GetNodeUnknownShapeStatus(node, is_dynamic); - if (is_dynamic) { - return ExecutorType::DYNAMIC_SUBGRAPH; + if (subgraph != nullptr) { + for (const auto &node : subgraph->GetDirectNode()) { + bool is_unknown_shape = false; + (void)AttrUtils::GetBool(node->GetOpDesc(), kOwnerGraphIsUnknown, is_unknown_shape); + if (is_unknown_shape) { + return ExecutorType::DYNAMIC_SUBGRAPH; + } else { + return ExecutorType::COMPILED_SUBGRAPH; + } + } } - return ExecutorType::COMPILED_SUBGRAPH; } // rts kernel store is assigned to NetOutput diff --git a/inc/framework/common/ge_types.h b/inc/framework/common/ge_types.h index 91815fc2..fb1f0be1 100644 --- a/inc/framework/common/ge_types.h +++ b/inc/framework/common/ge_types.h @@ -28,16 +28,9 @@ #include "external/ge/ge_api_types.h" namespace ge { -enum RuntimeType { - HOST = 0, - DEVICE = 1 -}; +enum RuntimeType { HOST = 0, DEVICE = 1 }; -enum PerfLevel { - GEN_TASK_WITH_FUSION = -1, - GEN_TASK_WITHOUT_L2FUSION = 3, - GEN_TASK_WITHOUT_FUSION = 4 -}; +enum PerfLevel { GEN_TASK_WITH_FUSION = -1, GEN_TASK_WITHOUT_L2FUSION = 3, GEN_TASK_WITHOUT_FUSION = 4 }; enum FrameworkType { CAFFE = 0, @@ -55,12 +48,7 @@ enum OpEngineType { ENGINE_AIVECTOR = 4 // not support }; -enum InputAippType{ - DATA_WITHOUT_AIPP = 0, - DATA_WITH_STATIC_AIPP, - DATA_WITH_DYNAMIC_AIPP, - DYNAMIC_AIPP_NODE -}; +enum InputAippType { DATA_WITHOUT_AIPP = 0, DATA_WITH_STATIC_AIPP, DATA_WITH_DYNAMIC_AIPP, DYNAMIC_AIPP_NODE }; const char *const GE_ENGINE_ATTR_MEM_TYPE_HBM = "HBM"; const char *const GE_OPTION_EXEC_PLACEMENT = "ge.exec.placement"; @@ -107,7 +95,7 @@ struct OutputData { struct Command { std::string cmd_type; // Command type std::vector cmd_params; // Command params - uint64_t module_index; // prof module + uint64_t module_index; // prof module }; // The definition of I/O shape description @@ -117,6 +105,7 @@ struct ShapeDescription { int64_t height = 0; int64_t width = 0; std::vector dims; + std::vector> shape_ranges; }; // Definition of input and output description information diff --git a/metadef b/metadef index ba04e25e..1b09ed04 160000 --- a/metadef +++ b/metadef @@ -1 +1 @@ -Subproject commit ba04e25e878af2ac5f9a697806daee0768ae3bad +Subproject commit 1b09ed04b6dd22d1aed1bee92fd42736c0fafc65 diff --git a/parser b/parser index 308e3587..3d49906d 160000 --- a/parser +++ b/parser @@ -1 +1 @@ -Subproject commit 308e3587ec54fdd32ed7113d64a1335208701f59 +Subproject commit 3d49906d119b1cc01f4256d7992759ce9f3dcfcd