| @@ -441,8 +441,8 @@ Status AicpuTfNodeTask::EnsureSessionCreated(uint64_t session_id) { | |||||
| return SUCCESS; | return SUCCESS; | ||||
| } | } | ||||
| Status AicpuTfNodeTask::ReadResultSummaryAndPrepareMemory(TaskContext &context, | |||||
| std::vector<std::unique_ptr<TensorBuffer>> &out_shape_hbm) { | |||||
| Status AicpuNodeTaskBase::ReadResultSummaryAndPrepareMemory(TaskContext &context, | |||||
| std::vector<std::unique_ptr<TensorBuffer>> &out_shape_hbm) { | |||||
| for (auto i = 0; i < node_item_->num_outputs; ++i) { | for (auto i = 0; i < node_item_->num_outputs; ++i) { | ||||
| auto &result_summary = output_summary_host_[i]; | auto &result_summary = output_summary_host_[i]; | ||||
| GE_CHK_RT_RET(rtMemcpy(&result_summary, sizeof(aicpu::FWKAdapter::ResultSummary), | GE_CHK_RT_RET(rtMemcpy(&result_summary, sizeof(aicpu::FWKAdapter::ResultSummary), | ||||
| @@ -467,6 +467,31 @@ Status AicpuTfNodeTask::ReadResultSummaryAndPrepareMemory(TaskContext &context, | |||||
| return SUCCESS; | return SUCCESS; | ||||
| } | } | ||||
| Status AicpuNodeTask::CopyDataToHbm(TaskContext &context, | |||||
| const std::vector<std::unique_ptr<TensorBuffer>> &out_shape_hbm) { | |||||
| GE_CHK_BOOL_RET_STATUS(out_shape_hbm.size() == static_cast<std::size_t>(node_item_->num_outputs), | |||||
| INTERNAL_ERROR, | |||||
| "[Check][Size]Node[%s] has %d outputs but out shape is %zu not equal.", | |||||
| node_name_.c_str(), node_item_->num_outputs, out_shape_hbm.size()); | |||||
| GE_CHK_STATUS_RET_NOLOG(PrepareCopyInputs(context, out_shape_hbm)); | |||||
| RECORD_CALLBACK_EVENT(context.GetExecutionContext(), node_name_.c_str(), "[LaunchCopy] Start"); | |||||
| uint32_t flag = RT_KERNEL_DEFAULT; | |||||
| auto rt_ret = rtCpuKernelLaunchWithFlag(reinterpret_cast<const void *>(memcpy_so_name_.c_str()), | |||||
| reinterpret_cast<const void *>(memcpy_kernel_name_.c_str()), | |||||
| 1, // default core dim is 1 | |||||
| memcpy_args_.get(), memcpy_args_size_, | |||||
| nullptr, context.GetStream(), flag); | |||||
| GE_CHK_RT_RET(rt_ret); | |||||
| RECORD_CALLBACK_EVENT(context.GetExecutionContext(), node_name_.c_str(), "[LaunchCopy] End"); | |||||
| GE_CHK_RT_RET(rtStreamSynchronize(context.GetStream())); | |||||
| RECORD_CALLBACK_EVENT(context.GetExecutionContext(), node_name_.c_str(), "[SynchronizeCopy] End"); | |||||
| return SUCCESS; | |||||
| } | |||||
| Status AicpuTfNodeTask::CopyDataToHbm(TaskContext &context, | Status AicpuTfNodeTask::CopyDataToHbm(TaskContext &context, | ||||
| const std::vector<std::unique_ptr<TensorBuffer>> &out_shape_hbm) { | const std::vector<std::unique_ptr<TensorBuffer>> &out_shape_hbm) { | ||||
| GE_CHK_BOOL_RET_STATUS(out_shape_hbm.size() == static_cast<std::size_t>(node_item_->num_outputs), | GE_CHK_BOOL_RET_STATUS(out_shape_hbm.size() == static_cast<std::size_t>(node_item_->num_outputs), | ||||
| @@ -486,8 +511,8 @@ Status AicpuTfNodeTask::CopyDataToHbm(TaskContext &context, | |||||
| return SUCCESS; | return SUCCESS; | ||||
| } | } | ||||
| Status AicpuTfNodeTask::PrepareCopyInputs(const TaskContext &context, | |||||
| const std::vector<std::unique_ptr<TensorBuffer>> &out_shape_hbm) { | |||||
| Status AicpuNodeTaskBase::PrepareCopyInputs(const TaskContext &context, | |||||
| const std::vector<std::unique_ptr<TensorBuffer>> &out_shape_hbm) { | |||||
| std::vector<uint64_t> copy_input_release_flag; | std::vector<uint64_t> copy_input_release_flag; | ||||
| std::vector<uint64_t> copy_input_data_size; | std::vector<uint64_t> copy_input_data_size; | ||||
| std::vector<uint64_t> copy_input_src; | std::vector<uint64_t> copy_input_src; | ||||
| @@ -528,8 +553,8 @@ Status AicpuTfNodeTask::PrepareCopyInputs(const TaskContext &context, | |||||
| return SUCCESS; | return SUCCESS; | ||||
| } | } | ||||
| Status AicpuTfNodeTask::UpdateShapeByHbmBuffer(TaskContext &context, | |||||
| const std::vector<std::unique_ptr<TensorBuffer>> &out_shape_hbm) { | |||||
| Status AicpuNodeTaskBase::UpdateShapeByHbmBuffer(TaskContext &context, | |||||
| const std::vector<std::unique_ptr<TensorBuffer>> &out_shape_hbm) { | |||||
| GE_CHK_BOOL_RET_STATUS(out_shape_hbm.size() == static_cast<std::size_t>(node_item_->num_outputs), | GE_CHK_BOOL_RET_STATUS(out_shape_hbm.size() == static_cast<std::size_t>(node_item_->num_outputs), | ||||
| INTERNAL_ERROR, | INTERNAL_ERROR, | ||||
| "Node[%s] has %d outputs but out shape is %zu", | "Node[%s] has %d outputs but out shape is %zu", | ||||
| @@ -560,7 +585,7 @@ Status AicpuTfNodeTask::UpdateShapeByHbmBuffer(TaskContext &context, | |||||
| return SUCCESS; | return SUCCESS; | ||||
| } | } | ||||
| Status AicpuTfNodeTask::UpdateShapeAndDataByResultSummary(TaskContext &context) { | |||||
| Status AicpuNodeTaskBase::UpdateShapeAndDataByResultSummary(TaskContext &context) { | |||||
| GELOGD("Node[%s] update shape and data by result summary begin.", node_name_.c_str()); | GELOGD("Node[%s] update shape and data by result summary begin.", node_name_.c_str()); | ||||
| std::vector<std::unique_ptr<TensorBuffer>> out_shape_hbm; | std::vector<std::unique_ptr<TensorBuffer>> out_shape_hbm; | ||||
| @@ -649,7 +674,7 @@ Status AicpuTfNodeTask::LaunchTask(TaskContext &context) { | |||||
| return SUCCESS; | return SUCCESS; | ||||
| } | } | ||||
| Status AicpuTfNodeTask::TaskCallback(TaskContext &context) { | |||||
| Status AicpuNodeTaskBase::TaskCallback(TaskContext &context) { | |||||
| GELOGD("Node[%s] task callback start. is_dynamic=%s, unknown_type=%d.", | GELOGD("Node[%s] task callback start. is_dynamic=%s, unknown_type=%d.", | ||||
| node_name_.c_str(), node_item_->is_dynamic ? "true" : "false", unknown_type_); | node_name_.c_str(), node_item_->is_dynamic ? "true" : "false", unknown_type_); | ||||
| Status callback_ret = SUCCESS; | Status callback_ret = SUCCESS; | ||||
| @@ -666,14 +691,110 @@ Status AicpuTfNodeTask::TaskCallback(TaskContext &context) { | |||||
| return callback_ret; | return callback_ret; | ||||
| } | } | ||||
| Status AicpuNodeTask::SetMemCopyTask(const domi::TaskDef &task_def) { | |||||
| if (node_item_->num_outputs == 0) { | |||||
| GELOGD("Node[%s] type[%s] has no output, no need set mem_copy task.", | |||||
| node_name_.c_str(), node_item_->node_type.c_str()); | |||||
| return SUCCESS; | |||||
| } | |||||
| GELOGD("Start to set memcpy task for node[%s].", node_name_.c_str()); | |||||
| const domi::KernelDef &kernel_def = task_def.kernel(); | |||||
| auto &memcpy_args = kernel_def.args(); | |||||
| memcpy_args_size_ = kernel_def.args_size(); | |||||
| memcpy_so_name_ = kernel_def.so_name(); | |||||
| memcpy_kernel_name_ = kernel_def.kernel_name(); | |||||
| GE_IF_BOOL_EXEC(memcpy_args.size() != memcpy_args_size_, | |||||
| REPORT_INNER_ERROR("E19999", "MemCopy task def args.size=%zu, but args_size=%u not equal.", | |||||
| memcpy_args.size(), memcpy_args_size_); | |||||
| GELOGE(FAILED, "[Check][Size]MemCopy task def args.size=%zu, but args_size=%u not equal.", | |||||
| memcpy_args.size(), memcpy_args_size_); | |||||
| return FAILED;); | |||||
| GE_IF_BOOL_EXEC(memcpy_args_size_ < sizeof(aicpu::AicpuParamHead), | |||||
| REPORT_INNER_ERROR("E19999", | |||||
| "Task def args_size=%u is less than aicpu param head len=%zu.", | |||||
| memcpy_args_size_, sizeof(aicpu::AicpuParamHead)); | |||||
| GELOGE(FAILED, | |||||
| "[Check][Size] Task def args_size=%u is less than aicpu param head len=%zu.", | |||||
| memcpy_args_size_, sizeof(aicpu::AicpuParamHead)); | |||||
| return FAILED;); | |||||
| memcpy_args_.reset(new(std::nothrow) uint8_t[memcpy_args_size_]()); | |||||
| GE_IF_BOOL_EXEC(memcpy_args_ == nullptr, | |||||
| REPORT_INNER_ERROR("E19999", "new memory failed for Node[MemCopy], task_size[%u].", | |||||
| memcpy_args_size_); | |||||
| GELOGE(FAILED, "[Malloc][Memory] failed for Node[MemCopy], task_size[%u].", | |||||
| memcpy_args_size_); | |||||
| return FAILED;); | |||||
| errno_t sec_ret = memcpy_s(memcpy_args_.get(), memcpy_args_size_, memcpy_args.c_str(), memcpy_args.size()); | |||||
| GE_IF_BOOL_EXEC(sec_ret != EOK, | |||||
| REPORT_INNER_ERROR("E19999", | |||||
| "memcpy_s argc_ failed for Node[MemCopy], ret: %d", sec_ret); | |||||
| GELOGE(INTERNAL_ERROR, | |||||
| "[Update][args] failed for Node[MemCopy], ret: %d", sec_ret); | |||||
| return sec_ret;); | |||||
| auto memcpy_param_head = reinterpret_cast<aicpu::AicpuParamHead *>(memcpy_args_.get()); | |||||
| uint32_t memcpy_io_num = memcpy_param_head->ioAddrNum; | |||||
| auto memcpy_io_addr = memcpy_args_.get() + sizeof(aicpu::AicpuParamHead); | |||||
| // if has input and output, need copy to ioaddr | |||||
| int cpy_ret = memcpy_s(memcpy_io_addr, memcpy_args_size_ - sizeof(aicpu::AicpuParamHead), | |||||
| ©_io_addr_[0], sizeof(uint64_t) * memcpy_io_num); | |||||
| GE_IF_BOOL_EXEC(cpy_ret != 0, | |||||
| REPORT_INNER_ERROR("E19999", "Node[Memcpoy] memcpy io addr to AicpuParamHead failed," | |||||
| "ret=%d, args_size=%u, io nums=%u.", | |||||
| cpy_ret, memcpy_args_size_, memcpy_io_num); | |||||
| GELOGE(INTERNAL_ERROR, "[Update][io_addr]Node[MemCopy] memcpy io addr to AicpuParamHead failed," | |||||
| "ret=%d, args_size=%u, io nums=%u.", | |||||
| cpy_ret, memcpy_args_size_, memcpy_io_num); | |||||
| return INTERNAL_ERROR;); | |||||
| GELOGD("Set memcpy task for node[MemCopy] successfully."); | |||||
| return SUCCESS; | |||||
| } | |||||
| Status AicpuNodeTask::InitForDependComputeTask() { | |||||
| if ((unknown_type_ != DEPEND_COMPUTE) || (node_item_->num_outputs == 0)) { | |||||
| GELOGD("Node[%s] type[%s] unknown_type is %d, output num is %d.", | |||||
| node_name_.c_str(), node_item_->node_type.c_str(), unknown_type_, node_item_->num_outputs); | |||||
| return SUCCESS; | |||||
| } | |||||
| output_summary_.resize(node_item_->num_outputs); | |||||
| constexpr auto result_summary_size = sizeof(aicpu::FWKAdapter::ResultSummary); | |||||
| for (auto i = 0; i < node_item_->num_outputs; ++i) { | |||||
| GE_CHK_STATUS_RET(AllocTensorBuffer(result_summary_size, output_summary_[i]), | |||||
| "[Alloc][TensorBuffer] failed for Node[%s] to copy result summary info, size=%zu.", | |||||
| node_name_.c_str(), result_summary_size); | |||||
| } | |||||
| output_summary_host_.resize(node_item_->num_outputs); | |||||
| // init for mem copy task | |||||
| // copy task need copy output_data and output_shape, max len is 2 * output_num | |||||
| const size_t copy_input_buf_len = node_item_->num_outputs * 2 * sizeof(uint64_t); | |||||
| GE_CHK_STATUS_RET(AllocTensorBuffer(copy_input_buf_len, copy_input_release_flag_dev_), | |||||
| "[Alloc][TensorBuffer] failed for Node[%s] to copy task input release_flag, size=%zu", | |||||
| node_name_.c_str(), copy_input_buf_len); | |||||
| GE_CHK_STATUS_RET(AllocTensorBuffer(copy_input_buf_len, copy_input_data_size_dev_), | |||||
| "[Alloc][TensorBuffer] failed for Node[%s] to copy task input data_size, size=%zu", | |||||
| node_name_.c_str(), copy_input_buf_len); | |||||
| GE_CHK_STATUS_RET(AllocTensorBuffer(copy_input_buf_len, copy_input_src_dev_), | |||||
| "[Alloc][TensorBuffer] failed for Node[%s] to copy task input src, size=%zu", | |||||
| node_name_.c_str(), copy_input_buf_len); | |||||
| GE_CHK_STATUS_RET(AllocTensorBuffer(copy_input_buf_len, copy_input_dst_dev_), | |||||
| "[Alloc][TensorBuffer] failed for Node[%s] to copy task input dst, size=%zu", | |||||
| node_name_.c_str(), copy_input_buf_len); | |||||
| copy_io_addr_.emplace_back(reinterpret_cast<uintptr_t>(copy_input_release_flag_dev_->GetData())); | |||||
| copy_io_addr_.emplace_back(reinterpret_cast<uintptr_t>(copy_input_data_size_dev_->GetData())); | |||||
| copy_io_addr_.emplace_back(reinterpret_cast<uintptr_t>(copy_input_src_dev_->GetData())); | |||||
| copy_io_addr_.emplace_back(reinterpret_cast<uintptr_t>(copy_input_dst_dev_->GetData())); | |||||
| return SUCCESS; | |||||
| } | |||||
| Status AicpuNodeTask::Init(const HybridModel &model) { | Status AicpuNodeTask::Init(const HybridModel &model) { | ||||
| auto node_name = node_name_; | auto node_name = node_name_; | ||||
| GELOGD("Node[%s] init start.", node_name.c_str()); | GELOGD("Node[%s] init start.", node_name.c_str()); | ||||
| GE_CHK_BOOL_RET_STATUS(unknown_type_ != DEPEND_COMPUTE, FAILED, | |||||
| "[Check][Type]Node[%s] unknown type[%d] is depend compute, it's not supported now.", | |||||
| node_name.c_str(), unknown_type_); | |||||
| GE_CHK_BOOL_RET_STATUS(task_def_.has_kernel(), FAILED, | GE_CHK_BOOL_RET_STATUS(task_def_.has_kernel(), FAILED, | ||||
| "[Check][task_def_]Node[%s] task def does not has kernel.", node_name.c_str()); | "[Check][task_def_]Node[%s] task def does not has kernel.", node_name.c_str()); | ||||
| auto &kernel_def = task_def_.kernel(); | auto &kernel_def = task_def_.kernel(); | ||||
| @@ -762,7 +883,10 @@ Status AicpuNodeTask::Init(const HybridModel &model) { | |||||
| uint64_t ext_session_id = model.GetSessionId(); | uint64_t ext_session_id = model.GetSessionId(); | ||||
| GE_CHK_STATUS_RET(InitExtInfo(kernel_ext_info, ext_session_id), | GE_CHK_STATUS_RET(InitExtInfo(kernel_ext_info, ext_session_id), | ||||
| "[Init][ExtInfo] failed for Node[%s].", node_name.c_str()); | "[Init][ExtInfo] failed for Node[%s].", node_name.c_str()); | ||||
| GE_CHK_STATUS_RET(InitForDependComputeTask(), | |||||
| "[Init][DependComputeTask] failed for Node[%s].", | |||||
| node_name_.c_str()); | |||||
| if (ext_info_addr_dev_ == nullptr) { | if (ext_info_addr_dev_ == nullptr) { | ||||
| aicpu_param_head->extInfoLength = 0; | aicpu_param_head->extInfoLength = 0; | ||||
| aicpu_param_head->extInfoAddr = 0; | aicpu_param_head->extInfoAddr = 0; | ||||
| @@ -770,7 +894,11 @@ Status AicpuNodeTask::Init(const HybridModel &model) { | |||||
| aicpu_param_head->extInfoLength = ext_info_addr_dev_->GetSize(); | aicpu_param_head->extInfoLength = ext_info_addr_dev_->GetSize(); | ||||
| aicpu_param_head->extInfoAddr = reinterpret_cast<uintptr_t>(ext_info_addr_dev_->GetData()); | aicpu_param_head->extInfoAddr = reinterpret_cast<uintptr_t>(ext_info_addr_dev_->GetData()); | ||||
| } | } | ||||
| auto task_defs = model.GetTaskDefs(node_item_->node); | |||||
| GE_CHECK_NOTNULL(task_defs); | |||||
| if (unknown_type_ == DEPEND_COMPUTE) { | |||||
| GE_CHK_STATUS_RET_NOLOG(SetMemCopyTask((*task_defs)[1])); | |||||
| } | |||||
| GELOGD("Node[%s] init end.", node_name.c_str()); | GELOGD("Node[%s] init end.", node_name.c_str()); | ||||
| return SUCCESS; | return SUCCESS; | ||||
| } | } | ||||
| @@ -786,13 +914,29 @@ Status AicpuNodeTask::UpdateIoAddr(TaskContext &context) { | |||||
| io_addrs.emplace_back(reinterpret_cast<uintptr_t>(inputData->GetData())); | io_addrs.emplace_back(reinterpret_cast<uintptr_t>(inputData->GetData())); | ||||
| } | } | ||||
| GE_CHK_STATUS_RET_NOLOG(context.AllocateOutputs()); | |||||
| for (auto j = 0; j < node_item_->num_outputs; ++j) { | |||||
| auto outputData = context.GetOutput(j); | |||||
| GE_CHECK_NOTNULL(outputData); | |||||
| GELOGD("Node[%s] output[%d] addr = %p, size = %zu", node_name_.c_str(), j, | |||||
| outputData->GetData(), outputData->GetSize()); | |||||
| io_addrs.emplace_back(reinterpret_cast<uintptr_t>(outputData->GetData())); | |||||
| // known shape or not depend compute | |||||
| if (!node_item_->is_dynamic || unknown_type_ != DEPEND_COMPUTE) { | |||||
| // unknown type 4 do this in call back. | |||||
| GE_CHK_STATUS_RET_NOLOG(context.AllocateOutputs()); | |||||
| for (auto j = 0; j < node_item_->num_outputs; ++j) { | |||||
| auto outputData = context.GetOutput(j); | |||||
| GE_CHECK_NOTNULL(outputData); | |||||
| GELOGD("Node[%s] output[%d] addr = %p, size = %zu", | |||||
| node_name_.c_str(), j, outputData->GetData(), outputData->GetSize()); | |||||
| io_addrs.emplace_back(reinterpret_cast<uintptr_t>(outputData->GetData())); | |||||
| } | |||||
| } else { | |||||
| // unknown type 4 use result summary update ioaddr. | |||||
| GELOGD("Node[%s] is depend compute node, use result summary as out addr.", node_name_.c_str()); | |||||
| GE_CHK_BOOL_RET_STATUS(output_summary_.size() == static_cast<std::size_t>(node_item_->num_outputs), | |||||
| INTERNAL_ERROR, | |||||
| "[Check][Size]Node[%s] has %d output but %zu output summary not equal.", | |||||
| node_name_.c_str(), node_item_->num_outputs, output_summary_.size()); | |||||
| for (auto j = 0; j < node_item_->num_outputs; ++j) { | |||||
| void *summary_addr = output_summary_[j]->GetData(); | |||||
| io_addrs.emplace_back(reinterpret_cast<uintptr_t>(summary_addr)); | |||||
| } | |||||
| } | } | ||||
| auto io_addr = args_.get() + sizeof(aicpu::AicpuParamHead); | auto io_addr = args_.get() + sizeof(aicpu::AicpuParamHead); | ||||
| @@ -830,23 +974,6 @@ Status AicpuNodeTask::LaunchTask(TaskContext &context) { | |||||
| return SUCCESS; | return SUCCESS; | ||||
| } | } | ||||
| Status AicpuNodeTask::TaskCallback(TaskContext &context) { | |||||
| GELOGD("Node[%s] task callback start, is_dynamic = %s, unknown_type=%d.", | |||||
| node_name_.c_str(), node_item_->is_dynamic ? "true" : "false", unknown_type_); | |||||
| Status callback_ret = SUCCESS; | |||||
| // check need update shape, call update shape. | |||||
| if (node_item_->is_dynamic && unknown_type_ == DEPEND_SHAPE_RANGE) { | |||||
| // check result | |||||
| callback_ret = UpdateOutputShapeFromExtInfo(context); | |||||
| } else { | |||||
| GELOGD("Node[%s] unknown shape type is %d no need update output shape.", | |||||
| node_name_.c_str(), unknown_type_); | |||||
| } | |||||
| GELOGD("Node[%s] task callback end.", node_name_.c_str()); | |||||
| return callback_ret; | |||||
| } | |||||
| Status AiCpuNodeExecutor::PrepareTask(NodeTask &task, TaskContext &context) const { | Status AiCpuNodeExecutor::PrepareTask(NodeTask &task, TaskContext &context) const { | ||||
| // malloc HBM memory at Init, here just update them | // malloc HBM memory at Init, here just update them | ||||
| RECORD_EXECUTION_EVENT(context.GetExecutionContext(), context.GetNodeName(), "[AiCpuNodeExecutorPrepareTask] Start"); | RECORD_EXECUTION_EVENT(context.GetExecutionContext(), context.GetNodeName(), "[AiCpuNodeExecutorPrepareTask] Start"); | ||||
| @@ -55,12 +55,34 @@ class AicpuNodeTaskBase : public NodeTask { | |||||
| virtual Status LaunchTask(TaskContext &context) = 0; | virtual Status LaunchTask(TaskContext &context) = 0; | ||||
| virtual Status TaskCallback(TaskContext &context) = 0; | |||||
| virtual Status InitForDependComputeTask() = 0; | |||||
| Status TaskCallback(TaskContext &context); | |||||
| virtual Status UpdateShapeAndDataByResultSummary(TaskContext &context); | |||||
| virtual Status UpdateIoAddr(TaskContext &context) = 0; | virtual Status UpdateIoAddr(TaskContext &context) = 0; | ||||
| static Status AllocTensorBuffer(size_t size, std::unique_ptr<TensorBuffer> &tensor_buffer); | static Status AllocTensorBuffer(size_t size, std::unique_ptr<TensorBuffer> &tensor_buffer); | ||||
| virtual Status CopyDataToHbm(TaskContext &context, | |||||
| const std::vector<std::unique_ptr<TensorBuffer>> &out_shape_hbm) = 0; | |||||
| /// | |||||
| /// read result summary and prepare copy task memory. | |||||
| /// @param context task context | |||||
| /// @param out_shape_hbm if scalar, TensorBuffer->data is null, size=0 | |||||
| /// @return SUCCESS:success other:failed | |||||
| /// | |||||
| Status ReadResultSummaryAndPrepareMemory(TaskContext &context, | |||||
| std::vector<std::unique_ptr<TensorBuffer>> &out_shape_hbm); | |||||
| Status UpdateShapeByHbmBuffer(TaskContext &context, | |||||
| const std::vector<std::unique_ptr<TensorBuffer>> &out_shape_hbm); | |||||
| Status PrepareCopyInputs(const TaskContext &context, | |||||
| const std::vector<std::unique_ptr<TensorBuffer>> &out_shape_hbm); | |||||
| protected: | protected: | ||||
| const NodeItem *node_item_; | const NodeItem *node_item_; | ||||
| // just reference. | // just reference. | ||||
| @@ -78,6 +100,15 @@ class AicpuNodeTaskBase : public NodeTask { | |||||
| // ext info addr, device mem | // ext info addr, device mem | ||||
| std::unique_ptr<TensorBuffer> ext_info_addr_dev_; | std::unique_ptr<TensorBuffer> ext_info_addr_dev_; | ||||
| std::vector<std::unique_ptr<TensorBuffer>> output_summary_; | |||||
| std::vector<aicpu::FWKAdapter::ResultSummary> output_summary_host_; | |||||
| std::unique_ptr<TensorBuffer> copy_input_release_flag_dev_; | |||||
| std::unique_ptr<TensorBuffer> copy_input_data_size_dev_; | |||||
| std::unique_ptr<TensorBuffer> copy_input_src_dev_; | |||||
| std::unique_ptr<TensorBuffer> copy_input_dst_dev_; | |||||
| bool need_sync_ = false; | |||||
| }; | }; | ||||
| class AicpuTfNodeTask : public AicpuNodeTaskBase { | class AicpuTfNodeTask : public AicpuNodeTaskBase { | ||||
| @@ -93,33 +124,15 @@ class AicpuTfNodeTask : public AicpuNodeTaskBase { | |||||
| Status LaunchTask(TaskContext &context) override; | Status LaunchTask(TaskContext &context) override; | ||||
| Status TaskCallback(TaskContext &context) override; | |||||
| Status UpdateIoAddr(TaskContext &context) override; | Status UpdateIoAddr(TaskContext &context) override; | ||||
| private: | |||||
| Status SetMemCopyTask(const domi::TaskDef &task_def); | |||||
| Status InitForDependComputeTask(); | |||||
| Status UpdateShapeAndDataByResultSummary(TaskContext &context); | |||||
| /// | |||||
| /// read result summary and prepare copy task memory. | |||||
| /// @param context task context | |||||
| /// @param out_shape_hbm if scalar, TensorBuffer->data is null, size=0 | |||||
| /// @return SUCCESS:success other:failed | |||||
| /// | |||||
| Status ReadResultSummaryAndPrepareMemory(TaskContext &context, | |||||
| std::vector<std::unique_ptr<TensorBuffer>> &out_shape_hbm); | |||||
| Status CopyDataToHbm(TaskContext &context, | Status CopyDataToHbm(TaskContext &context, | ||||
| const std::vector<std::unique_ptr<TensorBuffer>> &out_shape_hbm); | |||||
| const std::vector<std::unique_ptr<TensorBuffer>> &out_shape_hbm) override; | |||||
| Status UpdateShapeByHbmBuffer(TaskContext &context, | |||||
| const std::vector<std::unique_ptr<TensorBuffer>> &out_shape_hbm); | |||||
| Status InitForDependComputeTask() override; | |||||
| Status PrepareCopyInputs(const TaskContext &context, | |||||
| const std::vector<std::unique_ptr<TensorBuffer>> &out_shape_hbm); | |||||
| private: | |||||
| Status SetMemCopyTask(const domi::TaskDef &task_def); | |||||
| static Status EnsureSessionCreated(uint64_t session_id); | static Status EnsureSessionCreated(uint64_t session_id); | ||||
| static uint64_t GetStepIdAddr(const HybridModel &model); | static uint64_t GetStepIdAddr(const HybridModel &model); | ||||
| @@ -135,17 +148,8 @@ class AicpuTfNodeTask : public AicpuNodeTaskBase { | |||||
| // just used for depend DEPEND_COMPUTE op | // just used for depend DEPEND_COMPUTE op | ||||
| std::unique_ptr<TensorBuffer> copy_task_args_buf_; | std::unique_ptr<TensorBuffer> copy_task_args_buf_; | ||||
| std::vector<std::unique_ptr<TensorBuffer>> output_summary_; | |||||
| std::vector<aicpu::FWKAdapter::ResultSummary> output_summary_host_; | |||||
| std::unique_ptr<TensorBuffer> copy_ioaddr_dev_; | std::unique_ptr<TensorBuffer> copy_ioaddr_dev_; | ||||
| std::unique_ptr<TensorBuffer> copy_input_release_flag_dev_; | |||||
| std::unique_ptr<TensorBuffer> copy_input_data_size_dev_; | |||||
| std::unique_ptr<TensorBuffer> copy_input_src_dev_; | |||||
| std::unique_ptr<TensorBuffer> copy_input_dst_dev_; | |||||
| bool need_sync_ = false; | |||||
| std::unique_ptr<TensorBuffer> copy_workspace_buf_; | std::unique_ptr<TensorBuffer> copy_workspace_buf_; | ||||
| }; | }; | ||||
| @@ -162,16 +166,30 @@ class AicpuNodeTask : public AicpuNodeTaskBase { | |||||
| Status LaunchTask(TaskContext &context) override; | Status LaunchTask(TaskContext &context) override; | ||||
| Status TaskCallback(TaskContext &context) override; | |||||
| Status CopyDataToHbm(TaskContext &context, | |||||
| const std::vector<std::unique_ptr<TensorBuffer>> &out_shape_hbm) override; | |||||
| Status UpdateIoAddr(TaskContext &context) override; | Status UpdateIoAddr(TaskContext &context) override; | ||||
| Status InitForDependComputeTask() override; | |||||
| private: | |||||
| Status SetMemCopyTask(const domi::TaskDef &task_def); | |||||
| protected: | protected: | ||||
| // host mem | // host mem | ||||
| std::unique_ptr<uint8_t[]> args_; | std::unique_ptr<uint8_t[]> args_; | ||||
| // host memcpy mem | |||||
| std::unique_ptr<uint8_t[]> memcpy_args_; | |||||
| std::string memcpy_so_name_; | |||||
| std::string memcpy_kernel_name_; | |||||
| // args size | |||||
| uint32_t memcpy_args_size_ = 0; | |||||
| // args size | // args size | ||||
| uint32_t args_size_ = 0; | uint32_t args_size_ = 0; | ||||
| std::vector<uint64_t> copy_io_addr_; | |||||
| }; | }; | ||||
| class AiCpuNodeExecutor : public NodeExecutor { | class AiCpuNodeExecutor : public NodeExecutor { | ||||
| @@ -433,13 +433,11 @@ Status DynamicSingleOp::ExecuteAsync(const vector<GeTensorDesc> &input_desc, | |||||
| if (!inputs_size.empty()) { | if (!inputs_size.empty()) { | ||||
| StreamResource *stream_resource = SingleOpManager::GetInstance().GetResource(resource_id_, stream_); | StreamResource *stream_resource = SingleOpManager::GetInstance().GetResource(resource_id_, stream_); | ||||
| GE_CHK_STATUS_RET_NOLOG(UpdateInputsBufferAddr(stream_resource, stream_, inputs_size, update_buffers)); | GE_CHK_STATUS_RET_NOLOG(UpdateInputsBufferAddr(stream_resource, stream_, inputs_size, update_buffers)); | ||||
| GE_CHK_STATUS_RET_NOLOG(SetHostTensorValue(input_desc, input_buffers)); | |||||
| } | } | ||||
| if (hybrid_model_executor_ != nullptr) { | if (hybrid_model_executor_ != nullptr) { | ||||
| GELOGD("Execute multi-task dynamic single op by hybrid model executor"); | GELOGD("Execute multi-task dynamic single op by hybrid model executor"); | ||||
| if (!inputs_size.empty()) { | |||||
| GE_CHK_STATUS_RET_NOLOG(SetHostTensorValue(input_desc, input_buffers)); | |||||
| } | |||||
| hybrid::HybridModelExecutor::ExecuteArgs args; | hybrid::HybridModelExecutor::ExecuteArgs args; | ||||
| GE_CHK_STATUS_RET_NOLOG(InitHybridModelArgs(update_buffers, output_buffers, input_desc, args)); | GE_CHK_STATUS_RET_NOLOG(InitHybridModelArgs(update_buffers, output_buffers, input_desc, args)); | ||||
| @@ -333,7 +333,7 @@ Status SingleOpModel::BuildTaskList(StreamResource *stream_resource, SingleOp &s | |||||
| single_op.tasks_.emplace_back(tbe_task); | single_op.tasks_.emplace_back(tbe_task); | ||||
| } else if (kernel_type == ccKernelType::AI_CPU || kernel_type == ccKernelType::CUST_AI_CPU) { | } else if (kernel_type == ccKernelType::AI_CPU || kernel_type == ccKernelType::CUST_AI_CPU) { | ||||
| GELOGD("Building AICPU_CC task"); | GELOGD("Building AICPU_CC task"); | ||||
| OpTask *task = nullptr; | |||||
| AiCpuCCTask *task = nullptr; | |||||
| uint64_t singleop_kernel_id = aicpu_kernel_id++; | uint64_t singleop_kernel_id = aicpu_kernel_id++; | ||||
| GELOGI("Build singleOp CCTask, kernel_id = %lu", singleop_kernel_id); | GELOGI("Build singleOp CCTask, kernel_id = %lu", singleop_kernel_id); | ||||
| GE_CHK_STATUS_RET_NOLOG(BuildCpuKernelTask(task_def.kernel(), &task, singleop_kernel_id)); | GE_CHK_STATUS_RET_NOLOG(BuildCpuKernelTask(task_def.kernel(), &task, singleop_kernel_id)); | ||||
| @@ -489,7 +489,7 @@ Status SingleOpModel::BuildKernelExTask(const domi::KernelExDef &kernel_def, AiC | |||||
| return SUCCESS; | return SUCCESS; | ||||
| } | } | ||||
| Status SingleOpModel::BuildCpuKernelTask(const domi::KernelDef &kernel_def, OpTask **task, uint64_t kernel_id) { | |||||
| Status SingleOpModel::BuildCpuKernelTask(const domi::KernelDef &kernel_def, AiCpuCCTask **task, uint64_t kernel_id) { | |||||
| const auto &context = kernel_def.context(); | const auto &context = kernel_def.context(); | ||||
| auto iter = op_list_.find(context.op_index()); | auto iter = op_list_.find(context.op_index()); | ||||
| if (iter == op_list_.end()) { | if (iter == op_list_.end()) { | ||||
| @@ -611,10 +611,19 @@ Status SingleOpModel::BuildTaskListForDynamicOp(StreamResource *stream_resource, | |||||
| } else if (lib_name == kEngineNameAiCpu) { | } else if (lib_name == kEngineNameAiCpu) { | ||||
| const auto &task_def = task_defs[0]; | const auto &task_def = task_defs[0]; | ||||
| GELOGD("Building AICPU_CC task"); | GELOGD("Building AICPU_CC task"); | ||||
| OpTask *task = nullptr; | |||||
| AiCpuCCTask *task = nullptr; | |||||
| uint64_t dynamic_singleop_kernel_id = aicpu_kernel_id++; | uint64_t dynamic_singleop_kernel_id = aicpu_kernel_id++; | ||||
| GELOGI("Build dynamic singleOp CCTask, kernel_id = %lu", dynamic_singleop_kernel_id); | GELOGI("Build dynamic singleOp CCTask, kernel_id = %lu", dynamic_singleop_kernel_id); | ||||
| GE_CHK_STATUS_RET_NOLOG(BuildCpuKernelTask(task_def.kernel(), &task, dynamic_singleop_kernel_id)); | GE_CHK_STATUS_RET_NOLOG(BuildCpuKernelTask(task_def.kernel(), &task, dynamic_singleop_kernel_id)); | ||||
| if (task->GetUnknownType() == DEPEND_COMPUTE) { | |||||
| if (task_defs.size() < kNumTaskWithMemCpyTask) { | |||||
| GELOGE(ACL_ERROR_GE_PARAM_INVALID, "[Check][Task]The copy task of the fourth operator was not found."); | |||||
| REPORT_INNER_ERROR("E19999", "The copy task of the fourth operator was not found."); | |||||
| return ACL_ERROR_GE_PARAM_INVALID; | |||||
| } | |||||
| const TaskDef ©_task_def = task_defs[1]; | |||||
| GE_CHK_STATUS_RET_NOLOG(task->SetMemCopyTask(copy_task_def.kernel())); | |||||
| } | |||||
| task->SetModelArgs(model_name_, model_id_); | task->SetModelArgs(model_name_, model_id_); | ||||
| single_op.op_task_.reset(task); | single_op.op_task_.reset(task); | ||||
| } else if (lib_name == kEngineNameAiCpuTf) { | } else if (lib_name == kEngineNameAiCpuTf) { | ||||
| @@ -71,7 +71,7 @@ class SingleOpModel { | |||||
| Status BuildKernelTask(const domi::TaskDef &task_def, TbeOpTask **task); | Status BuildKernelTask(const domi::TaskDef &task_def, TbeOpTask **task); | ||||
| Status BuildAtomicTask(const domi::TaskDef &task_def, AtomicAddrCleanOpTask **task); | Status BuildAtomicTask(const domi::TaskDef &task_def, AtomicAddrCleanOpTask **task); | ||||
| Status BuildKernelExTask(const domi::KernelExDef &kernel_def, AiCpuTask **task, uint64_t kernel_id); | Status BuildKernelExTask(const domi::KernelExDef &kernel_def, AiCpuTask **task, uint64_t kernel_id); | ||||
| Status BuildCpuKernelTask(const domi::KernelDef &kernel_def, OpTask **task, uint64_t kernel_id); | |||||
| Status BuildCpuKernelTask(const domi::KernelDef &kernel_def, AiCpuCCTask **task, uint64_t kernel_id); | |||||
| static void ParseOpModelParams(ModelHelper &model_helper, SingleOpModelParam ¶m); | static void ParseOpModelParams(ModelHelper &model_helper, SingleOpModelParam ¶m); | ||||
| void ParseArgTable(OpTask *task, SingleOp &op); | void ParseArgTable(OpTask *task, SingleOp &op); | ||||
| @@ -102,11 +102,8 @@ Status AiCpuCCTaskBuilder::BuildTask(AiCpuCCTask &task, uint64_t kernel_id, cons | |||||
| return ret; | return ret; | ||||
| } | } | ||||
| GE_CHK_STATUS_RET(task.SetInputConst(), "[Set][InputConst] failed."); | GE_CHK_STATUS_RET(task.SetInputConst(), "[Set][InputConst] failed."); | ||||
| GE_CHK_STATUS_RET(task.InitForSummaryAndCopy(), "[Init][SummaryAndCopy] failed."); | |||||
| if (task.GetUnknownType() == DEPEND_COMPUTE) { | |||||
| GELOGE(FAILED, "[Get][UnknownType] is depend compute, it's not supported now."); | |||||
| return FAILED; | |||||
| } | |||||
| auto aicpu_param_head = reinterpret_cast<aicpu::AicpuParamHead *>(task.args_.get()); | auto aicpu_param_head = reinterpret_cast<aicpu::AicpuParamHead *>(task.args_.get()); | ||||
| if (task.ext_info_addr_dev_ != nullptr) { | if (task.ext_info_addr_dev_ != nullptr) { | ||||
| aicpu_param_head->extInfoLength = kernel_ext_info.size(); | aicpu_param_head->extInfoLength = kernel_ext_info.size(); | ||||
| @@ -293,9 +293,6 @@ Status TbeOpTask::UpdateNodeByShape(const vector<GeTensorDesc> &input_desc, cons | |||||
| } | } | ||||
| Status TbeOpTask::EnableDynamicSupport(const NodePtr &node, void *tiling_buffer, uint32_t max_tiling_size) { | Status TbeOpTask::EnableDynamicSupport(const NodePtr &node, void *tiling_buffer, uint32_t max_tiling_size) { | ||||
| node_ = node; | |||||
| tiling_buffer_ = tiling_buffer; | |||||
| max_tiling_size_ = max_tiling_size; | |||||
| if (tiling_buffer != nullptr) { | if (tiling_buffer != nullptr) { | ||||
| uintptr_t *arg_base = nullptr; | uintptr_t *arg_base = nullptr; | ||||
| size_t arg_num = 0; | size_t arg_num = 0; | ||||
| @@ -313,6 +310,9 @@ Status TbeOpTask::EnableDynamicSupport(const NodePtr &node, void *tiling_buffer, | |||||
| } | } | ||||
| arg_base[tiling_index] = reinterpret_cast<uintptr_t>(tiling_buffer); | arg_base[tiling_index] = reinterpret_cast<uintptr_t>(tiling_buffer); | ||||
| } | } | ||||
| node_ = node; | |||||
| tiling_buffer_ = tiling_buffer; | |||||
| max_tiling_size_ = max_tiling_size; | |||||
| return SUCCESS; | return SUCCESS; | ||||
| } | } | ||||
| @@ -481,25 +481,6 @@ void TbeOpTask::GetIoAddr(uintptr_t *&arg_base, size_t &arg_count) { | |||||
| } | } | ||||
| } | } | ||||
| Status AtomicAddrCleanOpTask::EnableDynamicSupport(const NodePtr &node, void *tiling_buffer, uint32_t max_tiling_size) { | |||||
| node_ = node; | |||||
| tiling_buffer_ = tiling_buffer; | |||||
| max_tiling_size_ = max_tiling_size; | |||||
| if (tiling_buffer != nullptr) { | |||||
| uintptr_t *arg_base = nullptr; | |||||
| size_t arg_num = 0; | |||||
| GetIoAddr(arg_base, arg_num); | |||||
| uint32_t tiling_index = atomic_output_indices_.size(); | |||||
| if (arg_num == 0 || arg_num < tiling_index) { | |||||
| GELOGE(ACL_ERROR_GE_INTERNAL_ERROR, "[Check][Size]Tiling index %u, arg number %zu is invalid.", | |||||
| tiling_index, arg_num); | |||||
| return ACL_ERROR_GE_INTERNAL_ERROR; | |||||
| } | |||||
| arg_base[tiling_index] = reinterpret_cast<uintptr_t>(tiling_buffer); | |||||
| } | |||||
| return SUCCESS; | |||||
| } | |||||
| Status AtomicAddrCleanOpTask::UpdateNodeByShape(const vector<GeTensorDesc> &input_desc, | Status AtomicAddrCleanOpTask::UpdateNodeByShape(const vector<GeTensorDesc> &input_desc, | ||||
| const vector<GeTensorDesc> &output_desc) { | const vector<GeTensorDesc> &output_desc) { | ||||
| return SUCCESS; | return SUCCESS; | ||||
| @@ -583,6 +564,17 @@ AiCpuBaseTask::~AiCpuBaseTask() { | |||||
| if (ext_info_addr_dev_ != nullptr) { | if (ext_info_addr_dev_ != nullptr) { | ||||
| (void)rtFree(ext_info_addr_dev_); | (void)rtFree(ext_info_addr_dev_); | ||||
| } | } | ||||
| FreeHbm(copy_input_release_flag_dev_); | |||||
| FreeHbm(copy_input_data_size_dev_); | |||||
| FreeHbm(copy_input_src_dev_); | |||||
| FreeHbm(copy_input_dst_dev_); | |||||
| for (auto summary : output_summary_) { | |||||
| FreeHbm(summary); | |||||
| } | |||||
| for (auto out_shape : out_shape_hbm_) { | |||||
| FreeHbm(out_shape); | |||||
| } | |||||
| } | } | ||||
| Status AiCpuBaseTask::SetExtInfoAndType(const std::string &kernel_ext_info, uint64_t kernel_id) { | Status AiCpuBaseTask::SetExtInfoAndType(const std::string &kernel_ext_info, uint64_t kernel_id) { | ||||
| @@ -795,17 +787,7 @@ AiCpuTask::~AiCpuTask() { | |||||
| FreeHbm(workspace_addr_); | FreeHbm(workspace_addr_); | ||||
| FreeHbm(copy_workspace_buf_); | FreeHbm(copy_workspace_buf_); | ||||
| FreeHbm(copy_ioaddr_dev_); | FreeHbm(copy_ioaddr_dev_); | ||||
| FreeHbm(copy_input_release_flag_dev_); | |||||
| FreeHbm(copy_input_data_size_dev_); | |||||
| FreeHbm(copy_input_src_dev_); | |||||
| FreeHbm(copy_input_dst_dev_); | |||||
| FreeHbm(copy_task_args_buf_); | FreeHbm(copy_task_args_buf_); | ||||
| for (auto summary : output_summary_) { | |||||
| FreeHbm(summary); | |||||
| } | |||||
| for (auto out_shape : out_shape_hbm_) { | |||||
| FreeHbm(out_shape); | |||||
| } | |||||
| } | } | ||||
| Status AiCpuTask::LaunchKernel(rtStream_t stream) { | Status AiCpuTask::LaunchKernel(rtStream_t stream) { | ||||
| @@ -835,7 +817,7 @@ Status AiCpuTask::LaunchKernel(rtStream_t stream) { | |||||
| return SUCCESS; | return SUCCESS; | ||||
| } | } | ||||
| Status AiCpuTask::PrepareCopyInputs(vector<DataBuffer> &outputs) { | |||||
| Status AiCpuBaseTask::PrepareCopyInputs(vector<DataBuffer> &outputs) { | |||||
| std::vector<uint64_t> copy_input_release_flag; | std::vector<uint64_t> copy_input_release_flag; | ||||
| std::vector<uint64_t> copy_input_data_size; | std::vector<uint64_t> copy_input_data_size; | ||||
| std::vector<uint64_t> copy_input_src; | std::vector<uint64_t> copy_input_src; | ||||
| @@ -876,7 +858,7 @@ Status AiCpuTask::PrepareCopyInputs(vector<DataBuffer> &outputs) { | |||||
| return SUCCESS; | return SUCCESS; | ||||
| } | } | ||||
| Status AiCpuTask::ReadResultSummaryAndPrepareMemory() { | |||||
| Status AiCpuBaseTask::ReadResultSummaryAndPrepareMemory() { | |||||
| for (size_t i = 0; i < num_outputs_; ++i) { | for (size_t i = 0; i < num_outputs_; ++i) { | ||||
| auto &result_summary = output_summary_host_[i]; | auto &result_summary = output_summary_host_[i]; | ||||
| @@ -903,7 +885,20 @@ Status AiCpuTask::CopyDataToHbm(vector<DataBuffer> &outputs, | |||||
| return SUCCESS; | return SUCCESS; | ||||
| } | } | ||||
| Status AiCpuTask::UpdateShapeByHbmBuffer(vector<GeTensorDesc> &output_desc) { | |||||
| Status AiCpuCCTask::CopyDataToHbm(vector<DataBuffer> &outputs, | |||||
| rtStream_t stream) { | |||||
| GE_CHK_STATUS_RET_NOLOG(PrepareCopyInputs(outputs)); | |||||
| auto ret = rtCpuKernelLaunchWithFlag(static_cast<const void *>(memcpy_so_name_.data()), | |||||
| static_cast<const void *>(memcpy_kernel_name_.data()), | |||||
| block_dim_, memcpy_args_.get(), static_cast<uint32_t>(memcpy_args_size_), | |||||
| nullptr, stream, dump_flag_); | |||||
| GE_CHK_RT_RET(ret); | |||||
| GE_CHK_RT_RET(rtStreamSynchronize(stream)); | |||||
| return SUCCESS; | |||||
| } | |||||
| Status AiCpuBaseTask::UpdateShapeByHbmBuffer(vector<GeTensorDesc> &output_desc) { | |||||
| for (size_t i = 0; i < num_outputs_; ++i) { | for (size_t i = 0; i < num_outputs_; ++i) { | ||||
| const auto &result_summary = output_summary_host_[i]; | const auto &result_summary = output_summary_host_[i]; | ||||
| std::vector<int64_t> shape_dims; | std::vector<int64_t> shape_dims; | ||||
| @@ -932,7 +927,7 @@ Status AiCpuTask::UpdateShapeByHbmBuffer(vector<GeTensorDesc> &output_desc) { | |||||
| } | } | ||||
| Status AiCpuTask::UpdateShapeAndDataByResultSummary(vector<GeTensorDesc> &output_desc, | |||||
| Status AiCpuBaseTask::UpdateShapeAndDataByResultSummary(vector<GeTensorDesc> &output_desc, | |||||
| vector<DataBuffer> &outputs, | vector<DataBuffer> &outputs, | ||||
| rtStream_t stream) { | rtStream_t stream) { | ||||
| if (num_outputs_ == 0) { | if (num_outputs_ == 0) { | ||||
| @@ -1122,8 +1117,91 @@ Status AiCpuCCTask::LaunchKernel(const std::vector<GeTensorDesc> &input_desc, | |||||
| if (unknown_type_ == DEPEND_SHAPE_RANGE) { | if (unknown_type_ == DEPEND_SHAPE_RANGE) { | ||||
| GE_CHK_RT_RET(rtStreamSynchronize(stream)); | GE_CHK_RT_RET(rtStreamSynchronize(stream)); | ||||
| GE_CHK_STATUS_RET_NOLOG(UpdateOutputShape(output_desc)); | GE_CHK_STATUS_RET_NOLOG(UpdateOutputShape(output_desc)); | ||||
| } else if (unknown_type_ == DEPEND_COMPUTE) { | |||||
| GE_CHK_RT_RET(rtStreamSynchronize(stream)); | |||||
| GE_CHK_STATUS_RET_NOLOG(UpdateShapeAndDataByResultSummary(output_desc, output_buffers, stream)); | |||||
| } | |||||
| return SUCCESS; | |||||
| } | |||||
| Status AiCpuCCTask::InitForSummaryAndCopy() { | |||||
| if (unknown_type_ != DEPEND_COMPUTE || num_outputs_ == 0) { | |||||
| GELOGI("Unknown_type is %d, output num is %zu.", unknown_type_, num_outputs_); | |||||
| return SUCCESS; | |||||
| } | |||||
| output_summary_.resize(num_outputs_); | |||||
| constexpr auto result_summary_size = sizeof(aicpu::FWKAdapter::ResultSummary); | |||||
| for (size_t i = 0; i < num_outputs_; ++i) { | |||||
| GE_CHK_RT_RET(rtMalloc(&output_summary_[i], result_summary_size, RT_MEMORY_HBM)); | |||||
| } | } | ||||
| output_summary_host_.resize(num_outputs_); | |||||
| const size_t copy_input_buf_len = num_outputs_ * kCopyNum * sizeof(uint64_t); | |||||
| GE_CHK_RT_RET(rtMalloc(©_input_release_flag_dev_, copy_input_buf_len, RT_MEMORY_HBM)); | |||||
| GE_CHK_RT_RET(rtMalloc(©_input_data_size_dev_, copy_input_buf_len, RT_MEMORY_HBM)); | |||||
| GE_CHK_RT_RET(rtMalloc(©_input_src_dev_, copy_input_buf_len, RT_MEMORY_HBM)); | |||||
| GE_CHK_RT_RET(rtMalloc(©_input_dst_dev_, copy_input_buf_len, RT_MEMORY_HBM)); | |||||
| copy_io_addr_.emplace_back(reinterpret_cast<uintptr_t>(copy_input_release_flag_dev_)); | |||||
| copy_io_addr_.emplace_back(reinterpret_cast<uintptr_t>(copy_input_data_size_dev_)); | |||||
| copy_io_addr_.emplace_back(reinterpret_cast<uintptr_t>(copy_input_src_dev_)); | |||||
| copy_io_addr_.emplace_back(reinterpret_cast<uintptr_t>(copy_input_dst_dev_)); | |||||
| return SUCCESS; | |||||
| } | |||||
| Status AiCpuCCTask::SetMemCopyTask(const domi::KernelDef &kernel_def) { | |||||
| auto &memcpy_args = kernel_def.args(); | |||||
| memcpy_args_size_ = kernel_def.args_size(); | |||||
| memcpy_so_name_ = kernel_def.so_name(); | |||||
| memcpy_kernel_name_ = kernel_def.kernel_name(); | |||||
| GE_IF_BOOL_EXEC(memcpy_args.size() != memcpy_args_size_, | |||||
| REPORT_INNER_ERROR("E19999", "MemCopy task def args.size=%zu, but args_size=%u not equal.", | |||||
| memcpy_args.size(), memcpy_args_size_); | |||||
| GELOGE(FAILED, "[Check][Size]MemCopy task def args.size=%zu, but args_size=%u not equal.", | |||||
| memcpy_args.size(), memcpy_args_size_); | |||||
| return FAILED;); | |||||
| GE_IF_BOOL_EXEC(memcpy_args_size_ < sizeof(aicpu::AicpuParamHead), | |||||
| REPORT_INNER_ERROR("E19999", | |||||
| "Task def args_size=%u is less than aicpu param head len=%zu.", | |||||
| memcpy_args_size_, sizeof(aicpu::AicpuParamHead)); | |||||
| GELOGE(FAILED, | |||||
| "[Check][Size] Task def args_size=%u is less than aicpu param head len=%zu.", | |||||
| memcpy_args_size_, sizeof(aicpu::AicpuParamHead)); | |||||
| return FAILED;); | |||||
| memcpy_args_.reset(new(std::nothrow) uint8_t[memcpy_args_size_]()); | |||||
| GE_IF_BOOL_EXEC(memcpy_args_ == nullptr, | |||||
| REPORT_INNER_ERROR("E19999", "new memory failed for Node[MemCopy], task_size[%u].", | |||||
| memcpy_args_size_); | |||||
| GELOGE(FAILED, "[Malloc][Memory] failed for Node[MemCopy], task_size[%u].", | |||||
| memcpy_args_size_); | |||||
| return FAILED;); | |||||
| errno_t sec_ret = memcpy_s(memcpy_args_.get(), memcpy_args_size_, memcpy_args.c_str(), memcpy_args.size()); | |||||
| GE_IF_BOOL_EXEC(sec_ret != EOK, | |||||
| REPORT_INNER_ERROR("E19999", | |||||
| "memcpy_s argc_ failed for Node[MemCopy], ret: %d", sec_ret); | |||||
| GELOGE(INTERNAL_ERROR, | |||||
| "[Update][args] failed for Node[MemCopy], ret: %d", sec_ret); | |||||
| return sec_ret;); | |||||
| auto memcpy_param_head = reinterpret_cast<aicpu::AicpuParamHead *>(memcpy_args_.get()); | |||||
| uint32_t memcpy_io_num = memcpy_param_head->ioAddrNum; | |||||
| auto memcpy_io_addr = memcpy_args_.get() + sizeof(aicpu::AicpuParamHead); | |||||
| // if has input and output, need copy to ioaddr | |||||
| int cpy_ret = memcpy_s(memcpy_io_addr, memcpy_args_size_ - sizeof(aicpu::AicpuParamHead), | |||||
| ©_io_addr_[0], sizeof(uint64_t) * memcpy_io_num); | |||||
| GE_IF_BOOL_EXEC(cpy_ret != 0, | |||||
| REPORT_INNER_ERROR("E19999", "Node[Memcpoy] memcpy io addr to AicpuParamHead failed," | |||||
| "ret=%d, args_size=%u, io nums=%u.", | |||||
| cpy_ret, memcpy_args_size_, memcpy_io_num); | |||||
| GELOGE(INTERNAL_ERROR, "[Update][io_addr]Node[MemCopy] memcpy io addr to AicpuParamHead failed," | |||||
| "ret=%d, args_size=%u, io nums=%u.", | |||||
| cpy_ret, memcpy_args_size_, memcpy_io_num); | |||||
| return INTERNAL_ERROR;); | |||||
| GELOGD("Set memcpy task for node[MemCopy] successfully."); | |||||
| return SUCCESS; | return SUCCESS; | ||||
| } | } | ||||
| @@ -97,7 +97,7 @@ class TbeOpTask : public OpTask { | |||||
| const void *GetArgs() const; | const void *GetArgs() const; | ||||
| size_t GetArgSize() const; | size_t GetArgSize() const; | ||||
| const std::string &GetStubName() const; | const std::string &GetStubName() const; | ||||
| virtual Status EnableDynamicSupport(const NodePtr &node, void *tiling_buffer, uint32_t max_tiling_size); | |||||
| Status EnableDynamicSupport(const NodePtr &node, void *tiling_buffer, uint32_t max_tiling_size); | |||||
| const std::string &GetTaskType() const override; | const std::string &GetTaskType() const override; | ||||
| void SetHandle(void *handle); | void SetHandle(void *handle); | ||||
| @@ -149,7 +149,6 @@ class TbeOpTask : public OpTask { | |||||
| class AtomicAddrCleanOpTask : public TbeOpTask { | class AtomicAddrCleanOpTask : public TbeOpTask { | ||||
| public: | public: | ||||
| Status InitAtomicAddrCleanIndices(); | Status InitAtomicAddrCleanIndices(); | ||||
| Status EnableDynamicSupport(const NodePtr &node, void *tiling_buffer, uint32_t max_tiling_size) override; | |||||
| private: | private: | ||||
| Status UpdateNodeByShape(const vector<GeTensorDesc> &input_desc, | Status UpdateNodeByShape(const vector<GeTensorDesc> &input_desc, | ||||
| @@ -157,8 +156,8 @@ class AtomicAddrCleanOpTask : public TbeOpTask { | |||||
| Status UpdateIoAddr(const vector<DataBuffer> &inputs, const vector<DataBuffer> &outputs) override; | Status UpdateIoAddr(const vector<DataBuffer> &inputs, const vector<DataBuffer> &outputs) override; | ||||
| Status UpdateTilingArgs(rtStream_t stream) override; | Status UpdateTilingArgs(rtStream_t stream) override; | ||||
| Status CalcTilingInfo(optiling::utils::OpRunInfo &run_info) override; | Status CalcTilingInfo(optiling::utils::OpRunInfo &run_info) override; | ||||
| std::vector<int> atomic_output_indices_; | std::vector<int> atomic_output_indices_; | ||||
| }; | }; | ||||
| class AiCpuBaseTask : public OpTask { | class AiCpuBaseTask : public OpTask { | ||||
| @@ -179,7 +178,17 @@ class AiCpuBaseTask : public OpTask { | |||||
| rtStream_t stream); | rtStream_t stream); | ||||
| Status UpdateOutputShape(vector<GeTensorDesc> &output_desc); | Status UpdateOutputShape(vector<GeTensorDesc> &output_desc); | ||||
| Status UpdateShapeToOutputDesc(const GeShape &shape_new, GeTensorDesc &output_desc); | Status UpdateShapeToOutputDesc(const GeShape &shape_new, GeTensorDesc &output_desc); | ||||
| Status UpdateShapeAndDataByResultSummary(vector<GeTensorDesc> &output_desc, | |||||
| vector<DataBuffer> &outputs, | |||||
| rtStream_t stream); | |||||
| Status ReadResultSummaryAndPrepareMemory(); | |||||
| Status PrepareCopyInputs(vector<DataBuffer> &outputs); | |||||
| Status UpdateShapeByHbmBuffer(vector<GeTensorDesc> &output_desc); | |||||
| virtual Status CopyDataToHbm(vector<DataBuffer> &outputs, rtStream_t stream) = 0; | |||||
| protected: | protected: | ||||
| size_t num_inputs_ = 0; | size_t num_inputs_ = 0; | ||||
| size_t num_outputs_ = 0; | size_t num_outputs_ = 0; | ||||
| @@ -187,6 +196,16 @@ class AiCpuBaseTask : public OpTask { | |||||
| std::unique_ptr<ge::hybrid::AicpuExtInfoHandler> aicpu_ext_handle_; | std::unique_ptr<ge::hybrid::AicpuExtInfoHandler> aicpu_ext_handle_; | ||||
| void *ext_info_addr_dev_ = nullptr; | void *ext_info_addr_dev_ = nullptr; | ||||
| vector<bool> input_is_const_; | vector<bool> input_is_const_; | ||||
| std::vector<void *> output_summary_; | |||||
| std::vector<aicpu::FWKAdapter::ResultSummary> output_summary_host_; | |||||
| void *copy_input_release_flag_dev_ = nullptr; | |||||
| void *copy_input_data_size_dev_ = nullptr; | |||||
| void *copy_input_src_dev_ = nullptr; | |||||
| void *copy_input_dst_dev_ = nullptr; | |||||
| vector<void *> out_shape_hbm_; | |||||
| }; | }; | ||||
| class AiCpuTask : public AiCpuBaseTask { | class AiCpuTask : public AiCpuBaseTask { | ||||
| @@ -207,16 +226,9 @@ class AiCpuTask : public AiCpuBaseTask { | |||||
| private: | private: | ||||
| // for copy task. | // for copy task. | ||||
| Status InitForSummaryAndCopy(); | Status InitForSummaryAndCopy(); | ||||
| Status UpdateShapeAndDataByResultSummary(vector<GeTensorDesc> &output_desc, | |||||
| vector<DataBuffer> &outputs, | |||||
| rtStream_t stream); | |||||
| Status ReadResultSummaryAndPrepareMemory(); | |||||
| Status CopyDataToHbm(vector<DataBuffer> &outputs, rtStream_t stream); | |||||
| Status PrepareCopyInputs(vector<DataBuffer> &outputs); | |||||
| Status UpdateShapeByHbmBuffer(vector<GeTensorDesc> &output_desc); | |||||
| Status CopyDataToHbm(vector<DataBuffer> &outputs, rtStream_t stream) override; | |||||
| private: | |||||
| friend class AiCpuTaskBuilder; | friend class AiCpuTaskBuilder; | ||||
| void *workspace_addr_ = nullptr; | void *workspace_addr_ = nullptr; | ||||
| std::string task_info_; | std::string task_info_; | ||||
| @@ -235,17 +247,7 @@ class AiCpuTask : public AiCpuBaseTask { | |||||
| void *copy_task_args_buf_ = nullptr; | void *copy_task_args_buf_ = nullptr; | ||||
| void *copy_workspace_buf_ = nullptr; | void *copy_workspace_buf_ = nullptr; | ||||
| std::vector<void *> output_summary_; | |||||
| std::vector<aicpu::FWKAdapter::ResultSummary> output_summary_host_; | |||||
| void *copy_ioaddr_dev_ = nullptr; | void *copy_ioaddr_dev_ = nullptr; | ||||
| void *copy_input_release_flag_dev_ = nullptr; | |||||
| void *copy_input_data_size_dev_ = nullptr; | |||||
| void *copy_input_src_dev_ = nullptr; | |||||
| void *copy_input_dst_dev_ = nullptr; | |||||
| vector<void *> out_shape_hbm_; | |||||
| uint64_t kernel_id_ = 0; | uint64_t kernel_id_ = 0; | ||||
| }; | }; | ||||
| @@ -264,13 +266,16 @@ class AiCpuCCTask : public AiCpuBaseTask { | |||||
| void SetkernelName(const std::string &kernel_Name); | void SetkernelName(const std::string &kernel_Name); | ||||
| void SetIoAddr(uintptr_t *io_addr); | void SetIoAddr(uintptr_t *io_addr); | ||||
| size_t GetArgSize() const; | size_t GetArgSize() const; | ||||
| Status SetMemCopyTask(const domi::KernelDef &kernel_def); | |||||
| Status LaunchKernel(const std::vector<GeTensorDesc> &input_desc, | Status LaunchKernel(const std::vector<GeTensorDesc> &input_desc, | ||||
| const std::vector<DataBuffer> &input_buffers, | const std::vector<DataBuffer> &input_buffers, | ||||
| std::vector<GeTensorDesc> &output_desc, | std::vector<GeTensorDesc> &output_desc, | ||||
| std::vector<DataBuffer> &output_buffers, | std::vector<DataBuffer> &output_buffers, | ||||
| rtStream_t stream) override; | rtStream_t stream) override; | ||||
| private: | |||||
| Status InitForSummaryAndCopy(); | |||||
| Status CopyDataToHbm(vector<DataBuffer> &outputs, rtStream_t stream) override; | |||||
| private: | private: | ||||
| friend class AiCpuCCTaskBuilder; | friend class AiCpuCCTaskBuilder; | ||||
| std::string so_name_; | std::string so_name_; | ||||
| @@ -284,6 +289,13 @@ private: | |||||
| uint32_t dump_flag_ = RT_KERNEL_DEFAULT; | uint32_t dump_flag_ = RT_KERNEL_DEFAULT; | ||||
| std::string op_type_; | std::string op_type_; | ||||
| uint64_t kernel_id_ = 0; | uint64_t kernel_id_ = 0; | ||||
| // host memcpy mem | |||||
| std::unique_ptr<uint8_t[]> memcpy_args_; | |||||
| std::string memcpy_so_name_; | |||||
| std::string memcpy_kernel_name_; | |||||
| std::vector<uint64_t> copy_io_addr_; | |||||
| // args size | |||||
| uint32_t memcpy_args_size_ = 0; | |||||
| }; | }; | ||||
| class MemcpyAsyncTask : public OpTask { | class MemcpyAsyncTask : public OpTask { | ||||
| @@ -425,7 +425,7 @@ Status TbeTaskBuilder::InitTilingInfo(TbeOpTask &task) { | |||||
| GELOGD("[%s] Done allocating tiling buffer, size=%ld.", op_desc_->GetName().c_str(), max_size); | GELOGD("[%s] Done allocating tiling buffer, size=%ld.", op_desc_->GetName().c_str(), max_size); | ||||
| } | } | ||||
| GE_CHK_STATUS_RET_NOLOG(task.EnableDynamicSupport(node_, tiling_buffer, static_cast<uint32_t>(max_size))); | |||||
| task.EnableDynamicSupport(node_, tiling_buffer, static_cast<uint32_t>(max_size)); | |||||
| return SUCCESS; | return SUCCESS; | ||||
| } | } | ||||
| @@ -165,4 +165,3 @@ TEST_F(UtestAicpuNodeExecutor, aicpu_tf_node_task) { | |||||
| } | } | ||||
| } // namespace ge | } // namespace ge | ||||
| @@ -237,27 +237,3 @@ TEST_F(UtestSingleOpTask, test_aicpu_task_update_io_addr) { | |||||
| ASSERT_EQ(ret, PARAM_INVALID); | ASSERT_EQ(ret, PARAM_INVALID); | ||||
| } | } | ||||
| } | } | ||||
| TEST_F(UtestSingleOpTask, test_dynamic_support) { | |||||
| auto graph = make_shared<ComputeGraph>("graph"); | |||||
| auto op_desc = make_shared<OpDesc>("Add", "Add"); | |||||
| auto node = graph->AddNode(op_desc); | |||||
| AtomicAddrCleanOpTask atomic_task; | |||||
| TbeOpTask tbe_task; | |||||
| tbe_task.arg_size_ = sizeof(void *) * 1; | |||||
| tbe_task.args_.reset(new (std::nothrow) uint8_t[tbe_task.arg_size_]); | |||||
| atomic_task.arg_size_ = sizeof(void *) * 1; | |||||
| atomic_task.args_.reset(new (std::nothrow) uint8_t[atomic_task.arg_size_]); | |||||
| ASSERT_EQ(tbe_task.EnableDynamicSupport(node, (void *)0x0001, 1), ACL_ERROR_GE_INTERNAL_ERROR); | |||||
| ASSERT_EQ(atomic_task.EnableDynamicSupport(node, (void *)0x0001, 1), ACL_ERROR_GE_INTERNAL_ERROR); | |||||
| tbe_task.arg_size_ = sizeof(void *) * 2; | |||||
| tbe_task.args_.reset(new (std::nothrow) uint8_t[tbe_task.arg_size_]); | |||||
| atomic_task.arg_size_ = sizeof(void *) * 2; | |||||
| atomic_task.args_.reset(new (std::nothrow) uint8_t[atomic_task.arg_size_]); | |||||
| ASSERT_EQ(tbe_task.EnableDynamicSupport(node, (void *)0x0001, 1), SUCCESS); | |||||
| ASSERT_EQ(atomic_task.EnableDynamicSupport(node, (void *)0x0001, 1), SUCCESS); | |||||
| tbe_task.tiling_buffer_ = nullptr; | |||||
| atomic_task.tiling_buffer_ = nullptr; | |||||
| } | |||||