@@ -69,6 +69,7 @@ | |||||
#include "graph/passes/link_gen_mask_nodes_pass.h" | #include "graph/passes/link_gen_mask_nodes_pass.h" | ||||
#include "graph/passes/mark_graph_unknown_status_pass.h" | #include "graph/passes/mark_graph_unknown_status_pass.h" | ||||
#include "graph/passes/merge_pass.h" | #include "graph/passes/merge_pass.h" | ||||
#include "graph/passes/merge_input_memcpy_pass.h" | |||||
#include "graph/passes/merge_to_stream_merge_pass.h" | #include "graph/passes/merge_to_stream_merge_pass.h" | ||||
#include "graph/passes/multi_batch_pass.h" | #include "graph/passes/multi_batch_pass.h" | ||||
#include "graph/passes/next_iteration_pass.h" | #include "graph/passes/next_iteration_pass.h" | ||||
@@ -1959,6 +1960,8 @@ Status GraphManager::OptimizeStage1(ge::ComputeGraphPtr &compute_graph) { | |||||
GELOGI("get ge.exec.variable_acc failed. set default value."); | GELOGI("get ge.exec.variable_acc failed. set default value."); | ||||
} | } | ||||
PassManager after_merge_passes; | PassManager after_merge_passes; | ||||
GE_CHK_STATUS_RET( | |||||
after_merge_passes.AddPass("OptimizeStage1_1::MergeInputMemcpyPass", new (std::nothrow) MergeInputMemcpyPass)); | |||||
GE_CHK_STATUS_RET( | GE_CHK_STATUS_RET( | ||||
after_merge_passes.AddPass("OptimizeStage1_1::SwitchDataEdgesBypass", new (std::nothrow) SwitchDataEdgesBypass)); | after_merge_passes.AddPass("OptimizeStage1_1::SwitchDataEdgesBypass", new (std::nothrow) SwitchDataEdgesBypass)); | ||||
GE_CHK_STATUS_RET( | GE_CHK_STATUS_RET( | ||||
@@ -0,0 +1,98 @@ | |||||
/** | |||||
* Copyright 2019-2020 Huawei Technologies Co., Ltd | |||||
* | |||||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||||
* you may not use this file except in compliance with the License. | |||||
* You may obtain a copy of the License at | |||||
* | |||||
* http://www.apache.org/licenses/LICENSE-2.0 | |||||
* | |||||
* Unless required by applicable law or agreed to in writing, software | |||||
* distributed under the License is distributed on an "AS IS" BASIS, | |||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
* See the License for the specific language governing permissions and | |||||
* limitations under the License. | |||||
*/ | |||||
#include "graph/passes/merge_input_memcpy_pass.h" | |||||
#include "common/ge/ge_util.h" | |||||
#include "ge/ge_api_types.h" | |||||
#include "graph/common/omg_util.h" | |||||
namespace ge { | |||||
Status MergeInputMemcpyPass::Run(ComputeGraphPtr graph) { | |||||
GELOGD("MergeInputMemcpyPass Enter"); | |||||
for (const auto &node : graph->GetDirectNode()) { | |||||
if ((node->GetType() != MERGE) && (node->GetType() != REFMERGE)) { | |||||
continue; | |||||
} | |||||
GE_CHECK_NOTNULL(node->GetOpDesc()); | |||||
GE_CHK_STATUS_RET(AddMemcpyAsyncNodes(graph, node, node->GetOpDesc()->HasAttr(ATTR_INSERT_BY_MBATCH)), | |||||
"Merge add memcpy node failed."); | |||||
} | |||||
GELOGD("MergeInputMemcpyPass Leave"); | |||||
return SUCCESS; | |||||
} | |||||
/// | |||||
/// @brief Add MemcpyAsync Op as Merge in_node | |||||
/// @param [in] graph | |||||
/// @param [in] node | |||||
/// @param [in] multi_batch_flag | |||||
/// @return Status | |||||
/// | |||||
Status MergeInputMemcpyPass::AddMemcpyAsyncNodes(const ComputeGraphPtr &graph, const NodePtr &node, | |||||
bool multi_batch_flag) { | |||||
for (const InDataAnchorPtr &in_data_anchor : node->GetAllInDataAnchors()) { | |||||
OutDataAnchorPtr peer_out_anchor = in_data_anchor->GetPeerOutAnchor(); | |||||
GE_IF_BOOL_EXEC(peer_out_anchor == nullptr, continue); | |||||
NodePtr in_node = peer_out_anchor->GetOwnerNode(); | |||||
const std::string &type = in_node->GetType(); | |||||
// For WhileLoop no need memcpy for merge. | |||||
GE_IF_BOOL_EXEC((type == ENTER) || (type == REFENTER) || (type == NEXTITERATION) || (type == REFNEXTITERATION), | |||||
continue); | |||||
const std::string &memcpy_name = node->GetName() + "_input_" + std::to_string(in_data_anchor->GetIdx()); | |||||
NodePtr memcpy_node = CreateMemcpyAsyncNode(graph, memcpy_name, peer_out_anchor, multi_batch_flag); | |||||
GE_CHK_BOOL_EXEC(memcpy_node != nullptr, return FAILED, "Create MemcpyAsync node failed."); | |||||
GE_CHK_STATUS(GraphUtils::RemoveEdge(peer_out_anchor, in_data_anchor), "MemcpyAsync node remove edge failed."); | |||||
GE_CHK_STATUS(GraphUtils::AddEdge(peer_out_anchor, memcpy_node->GetInDataAnchor(0)), | |||||
"MemcpyAsync node add edge failed."); | |||||
GE_CHK_STATUS(GraphUtils::AddEdge(memcpy_node->GetOutDataAnchor(0), in_data_anchor), | |||||
"MemcpyAsync node add edge failed."); | |||||
} | |||||
return SUCCESS; | |||||
} | |||||
/// | |||||
/// @brief Add MemcpyAsync Node | |||||
/// @param [in] graph | |||||
/// @param [in] name | |||||
/// @param [in] out_data_anchor | |||||
/// @param [in] multi_batch_flag | |||||
/// @return ge::NodePtr | |||||
/// | |||||
NodePtr MergeInputMemcpyPass::CreateMemcpyAsyncNode(const ComputeGraphPtr &graph, const std::string &name, | |||||
const OutDataAnchorPtr &out_data_anchor, bool multi_batch_flag) { | |||||
OpDescPtr pre_op_desc = out_data_anchor->GetOwnerNode()->GetOpDesc(); | |||||
GE_CHK_BOOL_EXEC(pre_op_desc != nullptr, return nullptr, "OpDesc of pre node is invalid."); | |||||
const std::string &memcpy_type = multi_batch_flag ? MEMCPYADDRASYNC : MEMCPYASYNC; | |||||
const std::string &node_name = name + "_" + memcpy_type; | |||||
GELOGI("Create MemcpyAsync op:%s.", node_name.c_str()); | |||||
OpDescPtr op_desc = MakeShared<OpDesc>(node_name, memcpy_type); | |||||
if (op_desc == nullptr) { | |||||
GELOGE(FAILED, "Create op_desc failed, MemcpyAsync:%s.", node_name.c_str()); | |||||
return nullptr; | |||||
} | |||||
GE_CHK_BOOL_EXEC(op_desc->AddInputDesc(pre_op_desc->GetOutputDesc(out_data_anchor->GetIdx())) == GRAPH_SUCCESS, | |||||
return nullptr, "Create MemcpyAsync op: add input desc failed."); | |||||
GE_CHK_BOOL_EXEC(op_desc->AddOutputDesc(pre_op_desc->GetOutputDesc(out_data_anchor->GetIdx())) == GRAPH_SUCCESS, | |||||
return nullptr, "Create MemcpyAsync op: add output desc failed."); | |||||
return graph->AddNode(op_desc); | |||||
} | |||||
} // namespace ge | |||||
@@ -0,0 +1,49 @@ | |||||
/** | |||||
* Copyright 2019-2020 Huawei Technologies Co., Ltd | |||||
* | |||||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||||
* you may not use this file except in compliance with the License. | |||||
* You may obtain a copy of the License at | |||||
* | |||||
* http://www.apache.org/licenses/LICENSE-2.0 | |||||
* | |||||
* Unless required by applicable law or agreed to in writing, software | |||||
* distributed under the License is distributed on an "AS IS" BASIS, | |||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
* See the License for the specific language governing permissions and | |||||
* limitations under the License. | |||||
*/ | |||||
#ifndef GE_GRAPH_PASSES_MERGE_ADD_INPUT_MEMCPY_PASS_H_ | |||||
#define GE_GRAPH_PASSES_MERGE_ADD_INPUT_MEMCPY_PASS_H_ | |||||
#include "inc/graph_pass.h" | |||||
namespace ge { | |||||
class MergeInputMemcpyPass : public GraphPass { | |||||
public: | |||||
Status Run(ComputeGraphPtr graph); | |||||
private: | |||||
/// | |||||
/// @brief Add MemcpyAsync Op as Merge in_node | |||||
/// @param [in] graph | |||||
/// @param [in] node | |||||
/// @param [in] multi_batch_flag | |||||
/// @return Status | |||||
/// | |||||
Status AddMemcpyAsyncNodes(const ComputeGraphPtr &graph, const NodePtr &node, bool multi_batch_flag); | |||||
/// | |||||
/// @brief Add MemcpyAsync Node | |||||
/// @param [in] graph | |||||
/// @param [in] name | |||||
/// @param [in] out_data_anchor | |||||
/// @param [in] multi_batch_flag | |||||
/// @return ge::NodePtr | |||||
/// | |||||
NodePtr CreateMemcpyAsyncNode(const ComputeGraphPtr &graph, const std::string &name, | |||||
const OutDataAnchorPtr &out_data_anchor, bool multi_batch_flag); | |||||
}; | |||||
} // namespace ge | |||||
#endif // GE_GRAPH_PASSES_MERGE_ADD_INPUT_MEMCPY_PASS_H_ |
@@ -32,7 +32,7 @@ Status MergeToStreamMergePass::Run(ComputeGraphPtr graph) { | |||||
OpDescPtr merge_op_desc = node->GetOpDesc(); | OpDescPtr merge_op_desc = node->GetOpDesc(); | ||||
GE_CHECK_NOTNULL(merge_op_desc); | GE_CHECK_NOTNULL(merge_op_desc); | ||||
if (merge_op_desc->HasAttr(ATTR_INSERT_BY_MBATCH)) { | if (merge_op_desc->HasAttr(ATTR_INSERT_BY_MBATCH)) { | ||||
GE_CHK_STATUS_RET(AddMemcpyAsyncNodes(graph, node, true), "Merge add memcpy node failed."); | |||||
GE_CHK_STATUS_RET(AddActiveNodes(graph, node, true), "Merge add active node failed."); | |||||
GE_CHK_STATUS_RET(SetStreamLabel(node, node->GetName()), "Set stream label failed"); | GE_CHK_STATUS_RET(SetStreamLabel(node, node->GetName()), "Set stream label failed"); | ||||
} else { | } else { | ||||
GE_CHK_STATUS_RET(ReplaceMergeNode(graph, node), "Add StreamMerge node failed."); | GE_CHK_STATUS_RET(ReplaceMergeNode(graph, node), "Add StreamMerge node failed."); | ||||
@@ -99,38 +99,23 @@ Status MergeToStreamMergePass::ReplaceMergeNode(const ComputeGraphPtr &graph, co | |||||
} | } | ||||
} | } | ||||
return AddMemcpyAsyncNodes(graph, stream_merge, false); | |||||
return AddActiveNodes(graph, stream_merge, false); | |||||
} | } | ||||
/// | /// | ||||
/// @brief Add MemcpyAsync Op as StreamMerge in_node | |||||
/// @brief Add StreamActive Op before StreamMerge/Merge | |||||
/// @param [in] graph | /// @param [in] graph | ||||
/// @param [in] node | /// @param [in] node | ||||
/// @param [in] multi_batch_flag | /// @param [in] multi_batch_flag | ||||
/// @return Status | /// @return Status | ||||
/// | /// | ||||
Status MergeToStreamMergePass::AddMemcpyAsyncNodes(const ComputeGraphPtr &graph, const NodePtr &node, | |||||
Status MergeToStreamMergePass::AddActiveNodes(const ComputeGraphPtr &graph, const NodePtr &node, | |||||
bool multi_batch_flag) { | bool multi_batch_flag) { | ||||
GE_CHK_BOOL_EXEC(node != nullptr, return FAILED, "Param of pre node is null."); | GE_CHK_BOOL_EXEC(node != nullptr, return FAILED, "Param of pre node is null."); | ||||
for (const InDataAnchorPtr &in_data_anchor : node->GetAllInDataAnchors()) { | for (const InDataAnchorPtr &in_data_anchor : node->GetAllInDataAnchors()) { | ||||
OutDataAnchorPtr peer_out_anchor = in_data_anchor->GetPeerOutAnchor(); | OutDataAnchorPtr peer_out_anchor = in_data_anchor->GetPeerOutAnchor(); | ||||
GE_IF_BOOL_EXEC(peer_out_anchor == nullptr, continue); | GE_IF_BOOL_EXEC(peer_out_anchor == nullptr, continue); | ||||
NodePtr in_node = peer_out_anchor->GetOwnerNode(); | |||||
const std::string &type = in_node->GetType(); | |||||
// For WhileLoop no need memcpy & active for merge. | |||||
GE_IF_BOOL_EXEC((type == ENTER) || (type == REFENTER) || (type == NEXTITERATION) || (type == REFNEXTITERATION), | |||||
continue); | |||||
const std::string &memcpy_name = node->GetName() + "_input_" + std::to_string(in_data_anchor->GetIdx()); | |||||
NodePtr memcpy_node = CreateMemcpyAsyncNode(graph, memcpy_name, peer_out_anchor, multi_batch_flag); | |||||
GE_CHK_BOOL_EXEC(memcpy_node != nullptr, return FAILED, "Create MemcpyAsync node failed."); | |||||
GE_CHK_STATUS(GraphUtils::RemoveEdge(peer_out_anchor, in_data_anchor), "MemcpyAsync node remove edge failed."); | |||||
GE_CHK_STATUS(GraphUtils::AddEdge(peer_out_anchor, memcpy_node->GetInDataAnchor(0)), | |||||
"MemcpyAsync node add edge failed."); | |||||
GE_CHK_STATUS(GraphUtils::AddEdge(memcpy_node->GetOutDataAnchor(0), in_data_anchor), | |||||
"MemcpyAsync node add edge failed."); | |||||
NodePtr active_node = CreateActiveNode(graph, memcpy_node); | |||||
NodePtr active_node = CreateActiveNode(graph, peer_out_anchor->GetOwnerNode()); | |||||
GE_CHK_BOOL_EXEC(active_node != nullptr, return FAILED, "Create StreamActive node failed."); | GE_CHK_BOOL_EXEC(active_node != nullptr, return FAILED, "Create StreamActive node failed."); | ||||
GE_CHK_STATUS(GraphUtils::AddEdge(active_node->GetOutControlAnchor(), node->GetInControlAnchor()), | GE_CHK_STATUS(GraphUtils::AddEdge(active_node->GetOutControlAnchor(), node->GetInControlAnchor()), | ||||
"StreamActive add ctrl edge failed."); | "StreamActive add ctrl edge failed."); | ||||
@@ -143,37 +128,6 @@ Status MergeToStreamMergePass::AddMemcpyAsyncNodes(const ComputeGraphPtr &graph, | |||||
return SUCCESS; | return SUCCESS; | ||||
} | } | ||||
/// | |||||
/// @brief Add MemcpyAsync Node | |||||
/// @param [in] graph | |||||
/// @param [in] name | |||||
/// @param [in] out_data_anchor | |||||
/// @param [in] multi_batch_flag | |||||
/// @return ge::NodePtr | |||||
/// | |||||
NodePtr MergeToStreamMergePass::CreateMemcpyAsyncNode(const ComputeGraphPtr &graph, const std::string &name, | |||||
const OutDataAnchorPtr &out_data_anchor, bool multi_batch_flag) { | |||||
GE_CHK_BOOL_EXEC(out_data_anchor != nullptr, return nullptr, "Param of input node is null."); | |||||
OpDescPtr pre_op_desc = out_data_anchor->GetOwnerNode()->GetOpDesc(); | |||||
GE_CHK_BOOL_EXEC(pre_op_desc != nullptr, return nullptr, "OpDesc of pre node is invalid."); | |||||
const std::string &memcpy_type = multi_batch_flag ? MEMCPYADDRASYNC : MEMCPYASYNC; | |||||
const std::string &node_name = name + "_" + memcpy_type; | |||||
GELOGI("Create MemcpyAsync op:%s.", node_name.c_str()); | |||||
OpDescPtr op_desc = MakeShared<OpDesc>(node_name, memcpy_type); | |||||
if (op_desc == nullptr) { | |||||
GELOGE(FAILED, "Create op_desc failed, MemcpyAsync:%s.", node_name.c_str()); | |||||
return nullptr; | |||||
} | |||||
GE_CHK_BOOL_EXEC(op_desc->AddInputDesc(pre_op_desc->GetOutputDesc(out_data_anchor->GetIdx())) == GRAPH_SUCCESS, | |||||
return nullptr, "Create MemcpyAsync op: add input desc failed."); | |||||
GE_CHK_BOOL_EXEC(op_desc->AddOutputDesc(pre_op_desc->GetOutputDesc(out_data_anchor->GetIdx())) == GRAPH_SUCCESS, | |||||
return nullptr, "Create MemcpyAsync op: add output desc failed."); | |||||
return graph->AddNode(op_desc); | |||||
} | |||||
/// | /// | ||||
/// @brief Create Active Op | /// @brief Create Active Op | ||||
/// @param [in] graph | /// @param [in] graph | ||||
@@ -193,7 +147,7 @@ NodePtr MergeToStreamMergePass::CreateActiveNode(const ComputeGraphPtr &graph, c | |||||
GE_CHK_BOOL_EXEC(active_node != nullptr, return nullptr, "Create StreamActive node failed."); | GE_CHK_BOOL_EXEC(active_node != nullptr, return nullptr, "Create StreamActive node failed."); | ||||
GE_IF_BOOL_EXEC(GraphUtils::AddEdge(node->GetOutControlAnchor(), active_node->GetInControlAnchor()) != SUCCESS, | GE_IF_BOOL_EXEC(GraphUtils::AddEdge(node->GetOutControlAnchor(), active_node->GetInControlAnchor()) != SUCCESS, | ||||
GELOGE(INTERNAL_ERROR, "add edge failed"); | GELOGE(INTERNAL_ERROR, "add edge failed"); | ||||
return nullptr); | |||||
return nullptr); | |||||
GE_IF_BOOL_EXEC(SetSwitchBranchNodeLabel(active_node, node_name) != SUCCESS, | GE_IF_BOOL_EXEC(SetSwitchBranchNodeLabel(active_node, node_name) != SUCCESS, | ||||
GELOGE(INTERNAL_ERROR, "set switch branch node label failed"); | GELOGE(INTERNAL_ERROR, "set switch branch node label failed"); | ||||
return nullptr); | return nullptr); | ||||
@@ -34,24 +34,13 @@ class MergeToStreamMergePass : public GraphPass { | |||||
Status ReplaceMergeNode(const ComputeGraphPtr &graph, const NodePtr &merge_node); | Status ReplaceMergeNode(const ComputeGraphPtr &graph, const NodePtr &merge_node); | ||||
/// | /// | ||||
/// @brief Add MemcpyAsync Op as StreamMerge in_node | |||||
/// @brief Add StreamActive Op as StreamMerge in_node | |||||
/// @param [in] graph | /// @param [in] graph | ||||
/// @param [in] node | /// @param [in] node | ||||
/// @param [in] multi_batch_flag | /// @param [in] multi_batch_flag | ||||
/// @return Status | /// @return Status | ||||
/// | /// | ||||
Status AddMemcpyAsyncNodes(const ComputeGraphPtr &graph, const NodePtr &node, bool multi_batch_flag); | |||||
/// | |||||
/// @brief Add MemcpyAsync Node | |||||
/// @param [in] graph | |||||
/// @param [in] name | |||||
/// @param [in] out_data_anchor | |||||
/// @param [in] multi_batch_flag | |||||
/// @return ge::NodePtr | |||||
/// | |||||
NodePtr CreateMemcpyAsyncNode(const ComputeGraphPtr &graph, const std::string &name, | |||||
const OutDataAnchorPtr &out_data_anchor, bool multi_batch_flag); | |||||
Status AddActiveNodes(const ComputeGraphPtr &graph, const NodePtr &node, bool multi_batch_flag); | |||||
/// | /// | ||||
/// @brief Create Active Op | /// @brief Create Active Op | ||||