@@ -71,7 +71,7 @@ if (ENABLE_OPEN_SRC) | |||||
set(STATIC_ACL_LIB ${GE_LIB_PATH}) | set(STATIC_ACL_LIB ${GE_LIB_PATH}) | ||||
find_module(slog libslog.so ${GE_LIB_PATH}) | find_module(slog libslog.so ${GE_LIB_PATH}) | ||||
find_module(static_mmpa libmmpa.a ${GE_LIB_PATH}) | find_module(static_mmpa libmmpa.a ${GE_LIB_PATH}) | ||||
find_module(msprof libmsprof.so ${GE_LIB_PATH}) | |||||
find_module(msprofiler libmsprofiler.a ${GE_LIB_PATH}) | |||||
find_module(hccl libhccl.so ${GE_LIB_PATH}) | find_module(hccl libhccl.so ${GE_LIB_PATH}) | ||||
find_module(adump_server libadump_server.a ${GE_LIB_PATH}) | find_module(adump_server libadump_server.a ${GE_LIB_PATH}) | ||||
find_module(runtime libruntime.so ${GE_LIB_PATH}) | find_module(runtime libruntime.so ${GE_LIB_PATH}) | ||||
@@ -80,20 +80,19 @@ if (ENABLE_OPEN_SRC) | |||||
find_module(error_manager liberror_manager.so ${GE_LIB_PATH}) | find_module(error_manager liberror_manager.so ${GE_LIB_PATH}) | ||||
find_module(ascend_hal_stub libascend_hal.so ${GE_LIB_PATH}) | find_module(ascend_hal_stub libascend_hal.so ${GE_LIB_PATH}) | ||||
find_module(error_manager_static liberror_manager.a ${GE_LIB_PATH}) | find_module(error_manager_static liberror_manager.a ${GE_LIB_PATH}) | ||||
find_module(msprofiler libmsprofiler.a ${GE_LIB_PATH}) | |||||
find_module(msprofiler_fwk libmsprofiler_fwk.a ${GE_LIB_PATH}) | |||||
#find_module(ascendcl_static libascendcl.a ${GE_LIB_PATH}) | #find_module(ascendcl_static libascendcl.a ${GE_LIB_PATH}) | ||||
else() | else() | ||||
find_module(slog libslog.so ${ASCEND_ATC_DIR}) | find_module(slog libslog.so ${ASCEND_ATC_DIR}) | ||||
find_module(static_mmpa libmmpa.a ${ASCEND_ATC_DIR}) | find_module(static_mmpa libmmpa.a ${ASCEND_ATC_DIR}) | ||||
find_module(error_manager liberror_manager.so ${ASCEND_ATC_DIR}) | find_module(error_manager liberror_manager.so ${ASCEND_ATC_DIR}) | ||||
if(PLATFORM STREQUAL "train") | if(PLATFORM STREQUAL "train") | ||||
find_module(msprof libmsprof.so ${ASCEND_DRIVER_COMMON_DIR}) | |||||
find_module(hccl libhccl.so ${ASCEND_RUNTIME_DIR}) | find_module(hccl libhccl.so ${ASCEND_RUNTIME_DIR}) | ||||
find_module(adump_server libadump_server.a ${ASCEND_RUNTIME_DIR}) | find_module(adump_server libadump_server.a ${ASCEND_RUNTIME_DIR}) | ||||
find_module(runtime libruntime.so ${ASCEND_RUNTIME_DIR}) | find_module(runtime libruntime.so ${ASCEND_RUNTIME_DIR}) | ||||
find_module(resource libresource.so ${ASCEND_RUNTIME_DIR}) | find_module(resource libresource.so ${ASCEND_RUNTIME_DIR}) | ||||
find_module(error_manager liberror_manager.so ${ASCEND_RUNTIME_DIR}) | find_module(error_manager liberror_manager.so ${ASCEND_RUNTIME_DIR}) | ||||
find_module(msprofiler libmsprofiler.a ${ASCEND_RUNTIME_DIR}) | |||||
find_module(msprofiler_fwk libmsprofiler_fwk.a ${ASCEND_RUNTIME_DIR}) | |||||
find_module(ascend_hal_stub libascend_hal.so ${ASCEND_DRIVER_DIR}/driver) | find_module(ascend_hal_stub libascend_hal.so ${ASCEND_DRIVER_DIR}/driver) | ||||
if(PRODUCT STREQUAL "flr3") | if(PRODUCT STREQUAL "flr3") | ||||
message(FATAL_ERROR "This platform is not supported in train mode, build terminated") | message(FATAL_ERROR "This platform is not supported in train mode, build terminated") | ||||
@@ -108,18 +107,15 @@ if (ENABLE_OPEN_SRC) | |||||
find_module(msprofiler libmsprofiler.a ${ASCEND_ACL_DIR}) | find_module(msprofiler libmsprofiler.a ${ASCEND_ACL_DIR}) | ||||
#find_module(ascendcl_static libascendcl.a ${ASCEND_ACL_DIR}) | #find_module(ascendcl_static libascendcl.a ${ASCEND_ACL_DIR}) | ||||
if(PRODUCT STREQUAL "flr3") | if(PRODUCT STREQUAL "flr3") | ||||
find_module(msprof libmsprof.so ${ASCEND_DRIVER_SHARE_DIR}) | |||||
elseif(PRODUCT STREQUAL "flr1") | elseif(PRODUCT STREQUAL "flr1") | ||||
find_module(ascend_hal_stub libascend_hal.so ${ASCEND_DRIVER_DIR}/driver) | find_module(ascend_hal_stub libascend_hal.so ${ASCEND_DRIVER_DIR}/driver) | ||||
find_module(msprof libmsprof.so ${ASCEND_DRIVER_COMMON_DIR}) | |||||
elseif(PRODUCT STREQUAL "flr2") | elseif(PRODUCT STREQUAL "flr2") | ||||
# flr2 ascend_hal_stub limsprof ? | # flr2 ascend_hal_stub limsprof ? | ||||
else() | else() | ||||
find_module(ascend_hal_stub libascend_hal.so ${ASCEND_DRIVER_DIR}) | find_module(ascend_hal_stub libascend_hal.so ${ASCEND_DRIVER_DIR}) | ||||
find_module(msprof libmsprof.so ${ASCEND_DRIVER_DIR}) | |||||
endif() | endif() | ||||
elseif(PLATFORM STREQUAL "all") | elseif(PLATFORM STREQUAL "all") | ||||
find_module(msprof libmsprof.so ${ASCEND_DRIVER_COMMON_DIR}) | |||||
find_module(msprofiler libmsprofiler.a ${ASCEND_DRIVER_COMMON_DIR}) | |||||
find_module(hccl libhccl.so ${ASCEND_RUNTIME_DIR}) | find_module(hccl libhccl.so ${ASCEND_RUNTIME_DIR}) | ||||
find_module(adump_server libadump_server.a ${ASCEND_ACL_DIR}) | find_module(adump_server libadump_server.a ${ASCEND_ACL_DIR}) | ||||
find_module(runtime libruntime.so ${ASCEND_ACL_DIR}) | find_module(runtime libruntime.so ${ASCEND_ACL_DIR}) | ||||
@@ -127,7 +123,7 @@ if (ENABLE_OPEN_SRC) | |||||
find_module(resource libresource.so ${ASCEND_ATC_DIR}) | find_module(resource libresource.so ${ASCEND_ATC_DIR}) | ||||
find_module(error_manager liberror_manager.so ${ASCEND_ATC_DIR}) | find_module(error_manager liberror_manager.so ${ASCEND_ATC_DIR}) | ||||
find_module(error_manager_static liberror_manager.a ${ASCEND_ACL_DIR}) | find_module(error_manager_static liberror_manager.a ${ASCEND_ACL_DIR}) | ||||
find_module(msprofiler libmsprofiler.a ${ASCEND_ACL_DIR}) | |||||
find_module(msprofiler_fwk libmsprofiler_fwk.a ${ASCEND_ACL_DIR}) | |||||
find_module(ascend_hal_stub libascend_hal.so ${ASCEND_DRIVER_DIR}/driver) | find_module(ascend_hal_stub libascend_hal.so ${ASCEND_DRIVER_DIR}/driver) | ||||
#find_module(ascendcl_static libascendcl.a ${ASCEND_ACL_DIR}) | #find_module(ascendcl_static libascendcl.a ${ASCEND_ACL_DIR}) | ||||
else() | else() | ||||
@@ -60,6 +60,8 @@ set(TRAIN_SRC_LIST | |||||
"common/dump/dump_manager.cc" | "common/dump/dump_manager.cc" | ||||
"common/dump/dump_properties.cc" | "common/dump/dump_properties.cc" | ||||
"common/dump/dump_op.cc" | "common/dump/dump_op.cc" | ||||
"common/profiling/ge_profiling.cc" | |||||
"common/profiling/ge_runner_profiling.cc" | |||||
"engine_manager/dnnengine_manager.cc" | "engine_manager/dnnengine_manager.cc" | ||||
"ge_local_engine/engine/host_cpu_engine.cc" | "ge_local_engine/engine/host_cpu_engine.cc" | ||||
"generator/ge_generator.cc" | "generator/ge_generator.cc" | ||||
@@ -332,7 +334,6 @@ set(TRAIN_SRC_LIST | |||||
"hybrid/hybrid_davinci_model.cc" | "hybrid/hybrid_davinci_model.cc" | ||||
"executor/ge_executor.cc" | "executor/ge_executor.cc" | ||||
"client/ge_api.cc" | "client/ge_api.cc" | ||||
"client/ge_prof.cc" | |||||
"analyzer/analyzer.cc" | "analyzer/analyzer.cc" | ||||
"ir_build/ge_ir_build.cc" | "ir_build/ge_ir_build.cc" | ||||
"ir_build/atc_ir_common.cc" | "ir_build/atc_ir_common.cc" | ||||
@@ -604,7 +605,7 @@ set(INFER_SRC_LIST | |||||
if (NOT ENABLE_D AND NOT ENABLE_ACL AND NOT ENABLE_MS_TESTCASES) | if (NOT ENABLE_D AND NOT ENABLE_ACL AND NOT ENABLE_MS_TESTCASES) | ||||
############ libge_runner.so ############ | ############ libge_runner.so ############ | ||||
add_library(ge_runner SHARED ${TRAIN_SRC_LIST} ${PROTO_SRCS} ${PROTO_CLIENT_SRCS}) | |||||
add_library(ge_runner SHARED ${TRAIN_SRC_LIST} ${PROTO_SRCS} ${PROTO_CLIENT_SRCS} $<TARGET_OBJECTS:msprofiler_fwk>) | |||||
target_compile_definitions(ge_runner PRIVATE | target_compile_definitions(ge_runner PRIVATE | ||||
PROTOBUF_INLINE_NOT_IN_HEADERS=0 | PROTOBUF_INLINE_NOT_IN_HEADERS=0 | ||||
@@ -649,7 +650,6 @@ target_link_libraries(ge_runner | |||||
$<BUILD_INTERFACE:intf_pub> | $<BUILD_INTERFACE:intf_pub> | ||||
ge_memory | ge_memory | ||||
adump_server | adump_server | ||||
msprofiler | |||||
static_mmpa | static_mmpa | ||||
-Wl,--no-as-needed | -Wl,--no-as-needed | ||||
graph | graph | ||||
@@ -658,7 +658,6 @@ target_link_libraries(ge_runner | |||||
register | register | ||||
c_sec | c_sec | ||||
slog | slog | ||||
msprof | |||||
runtime | runtime | ||||
resource | resource | ||||
error_manager | error_manager | ||||
@@ -783,7 +782,6 @@ target_link_libraries(opensrc_ascendcl PRIVATE | |||||
c_sec | c_sec | ||||
runtime | runtime | ||||
slog | slog | ||||
msprof | |||||
ascend_hal_stub | ascend_hal_stub | ||||
-Wl,--as-needed | -Wl,--as-needed | ||||
-lrt | -lrt | ||||
@@ -799,12 +797,10 @@ set_target_properties(opensrc_ascendcl PROPERTIES | |||||
add_custom_command( | add_custom_command( | ||||
OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/stub_ge_ir_build.cc | OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/stub_ge_ir_build.cc | ||||
${CMAKE_CURRENT_BINARY_DIR}/stub_ge_api.cc | ${CMAKE_CURRENT_BINARY_DIR}/stub_ge_api.cc | ||||
${CMAKE_CURRENT_BINARY_DIR}/stub_ge_prof.cc | |||||
COMMAND echo "Generating stub files." | COMMAND echo "Generating stub files." | ||||
&& ${HI_PYTHON} ${CMAKE_CURRENT_LIST_DIR}/stub/gen_stubapi.py ${GE_CODE_DIR}/inc/external ${CMAKE_CURRENT_BINARY_DIR} | && ${HI_PYTHON} ${CMAKE_CURRENT_LIST_DIR}/stub/gen_stubapi.py ${GE_CODE_DIR}/inc/external ${CMAKE_CURRENT_BINARY_DIR} | ||||
&& mv ge_ir_build.cc stub_ge_ir_build.cc | && mv ge_ir_build.cc stub_ge_ir_build.cc | ||||
&& mv ge_api.cc stub_ge_api.cc | && mv ge_api.cc stub_ge_api.cc | ||||
&& mv ge_prof.cc stub_ge_prof.cc | |||||
&& echo "Generating stub files end." | && echo "Generating stub files end." | ||||
#WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} | #WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} | ||||
#DEPENDS stub/gen_stubapi.py ${TOP_DIR}/inc/external ${CMAKE_CURRENT_BINARY_DIR} | #DEPENDS stub/gen_stubapi.py ${TOP_DIR}/inc/external ${CMAKE_CURRENT_BINARY_DIR} | ||||
@@ -813,7 +809,6 @@ add_custom_command( | |||||
add_custom_target(ge_stub | add_custom_target(ge_stub | ||||
DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/stub_ge_ir_build.cc | DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/stub_ge_ir_build.cc | ||||
${CMAKE_CURRENT_BINARY_DIR}/stub_ge_api.cc | ${CMAKE_CURRENT_BINARY_DIR}/stub_ge_api.cc | ||||
${CMAKE_CURRENT_BINARY_DIR}/stub_ge_prof.cc | |||||
) | ) | ||||
################################################################## | ################################################################## | ||||
@@ -855,7 +850,6 @@ target_include_directories(atc_stub_ge_compiler PRIVATE | |||||
############ stub/libge_runner.so ############ | ############ stub/libge_runner.so ############ | ||||
add_library(fwk_stub_ge_runner SHARED | add_library(fwk_stub_ge_runner SHARED | ||||
stub_ge_api.cc | stub_ge_api.cc | ||||
stub_ge_prof.cc | |||||
stub_ge_ir_build.cc | stub_ge_ir_build.cc | ||||
) | ) | ||||
@@ -1,369 +0,0 @@ | |||||
/** | |||||
* Copyright 2020 Huawei Technologies Co., Ltd | |||||
* | |||||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||||
* you may not use this file except in compliance with the License. | |||||
* You may obtain a copy of the License at | |||||
* | |||||
* http://www.apache.org/licenses/LICENSE-2.0 | |||||
* | |||||
* Unless required by applicable law or agreed to in writing, software | |||||
* distributed under the License is distributed on an "AS IS" BASIS, | |||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
* See the License for the specific language governing permissions and | |||||
* limitations under the License. | |||||
*/ | |||||
#include "ge/ge_prof.h" | |||||
#include "ge/ge_api.h" | |||||
#include "init/gelib.h" | |||||
#include "common/debug/log.h" | |||||
#include "framework/common/debug/ge_log.h" | |||||
#include "common/profiling/profiling_manager.h" | |||||
#include "graph/load/graph_loader.h" | |||||
#include "toolchain/prof_acl_api.h" | |||||
using std::map; | |||||
using std::string; | |||||
using std::vector; | |||||
namespace { | |||||
const uint32_t kMaxDeviceNum = 64; | |||||
const uint32_t kDeviceListIndex = 3; | |||||
const std::string kProfilingInit = "prof_init"; | |||||
const std::string kProfilingFinalize = "prof_finalize"; | |||||
const std::string kProfilingStart = "prof_start"; | |||||
const std::string kProfilingStop = "prof_stop"; | |||||
const std::string kDeviceNums = "devNums"; | |||||
const std::string kDeviceIdList = "devIdList"; | |||||
const std::string kAicoreMetrics = "aicoreMetrics"; | |||||
const std::map<ge::ProfilingAicoreMetrics, std::string> kProfAicoreMetricsToString = { | |||||
{ge::kAicoreArithmaticThroughput, "AICORE_ARITHMATIC_THROUGHPUT"}, | |||||
{ge::kAicorePipeline, "AICORE_PIPELINE"}, | |||||
{ge::kAicoreSynchronization, "AICORE_SYNCHRONIZATION"}, | |||||
{ge::kAicoreMemory, "AICORE_MEMORY"}, | |||||
{ge::kAicoreInternalMemory, "AICORE_INTERNAL_MEMORY"}, | |||||
{ge::kAicoreStall, "AICORE_STALL"}}; | |||||
} // namespace | |||||
static bool g_graph_prof_init_ = false; | |||||
static std::mutex g_prof_mutex_; | |||||
namespace ge { | |||||
struct aclgrphProfConfig { | |||||
ProfConfig config; | |||||
}; | |||||
Status aclgrphProfInit(const char *profiler_path, uint32_t length) { | |||||
GELOGT(TRACE_INIT, "Graph prof init start"); | |||||
std::shared_ptr<GELib> instance_ptr = ge::GELib::GetInstance(); | |||||
if (instance_ptr == nullptr || !instance_ptr->InitFlag()) { | |||||
GELOGE(GE_CLI_GE_NOT_INITIALIZED, "Ge client is not initialized."); | |||||
return FAILED; | |||||
} | |||||
std::lock_guard<std::mutex> lock(g_prof_mutex_); | |||||
if (g_graph_prof_init_) { | |||||
GELOGW("Multi graph profiling initializations."); | |||||
return GE_PROF_MULTI_INIT; | |||||
} | |||||
Status ret = CheckPath(profiler_path, length); | |||||
if (ret != SUCCESS) { | |||||
GELOGE(ret, "Profiling config path is invalid."); | |||||
return ret; | |||||
} | |||||
// if command mode is set, just return | |||||
if (ProfilingManager::Instance().ProfilingOn()) { | |||||
GELOGW("Graph prof init failed, cause profiling command pattern is running."); | |||||
return GE_PROF_MODE_CONFLICT; | |||||
} | |||||
ret = ProfInit(profiler_path); | |||||
if (ret != SUCCESS) { | |||||
GELOGE(ret, "ProfInit init fail"); | |||||
return ret; | |||||
} | |||||
GraphLoader graph_loader; | |||||
Command command; | |||||
command.cmd_params.clear(); | |||||
command.cmd_type = kProfilingInit; | |||||
command.module_index = PROF_MODEL_LOAD; | |||||
ret = graph_loader.CommandHandle(command); | |||||
if (ret != SUCCESS) { | |||||
GELOGE(ret, "Handle profiling command %s failed, config = %s", kProfilingInit.c_str(), profiler_path); | |||||
return ret; | |||||
} | |||||
if (!g_graph_prof_init_) { | |||||
g_graph_prof_init_ = true; | |||||
GELOGI("Profiling init successfully."); | |||||
} | |||||
GELOGI("Successfully execute GraphProfInit."); | |||||
return SUCCESS; | |||||
} | |||||
Status aclgrphProfFinalize() { | |||||
std::shared_ptr<GELib> instance_ptr = ge::GELib::GetInstance(); | |||||
if (instance_ptr == nullptr || !instance_ptr->InitFlag()) { | |||||
GELOGE(GE_CLI_GE_NOT_INITIALIZED, "Ge client is not initialized."); | |||||
return FAILED; | |||||
} | |||||
std::lock_guard<std::mutex> lock(g_prof_mutex_); | |||||
// if command mode is set, just return | |||||
if (ProfilingManager::Instance().ProfilingOn()) { | |||||
GELOGW("Graph prof finalize failed, cause profiling command pattern is running."); | |||||
return GE_PROF_MODE_CONFLICT; | |||||
} | |||||
if (!g_graph_prof_init_) { | |||||
GELOGE(GE_PROF_NOT_INIT, "Graph not profiling initialize."); | |||||
return GE_PROF_NOT_INIT; | |||||
} | |||||
GraphLoader graph_loader; | |||||
Command command; | |||||
command.cmd_params.clear(); | |||||
command.cmd_type = kProfilingFinalize; | |||||
Status ret = graph_loader.CommandHandle(command); | |||||
if (ret != SUCCESS) { | |||||
GELOGE(ret, "Handle profiling command %s failed.", kProfilingFinalize.c_str()); | |||||
return ret; | |||||
} | |||||
ret = ProfFinalize(); | |||||
if (ret != SUCCESS) { | |||||
GELOGE(ret, "Finalize profiling failed, result = %d", ret); | |||||
} | |||||
if (ret == SUCCESS) { | |||||
g_graph_prof_init_ = false; | |||||
GELOGI("Successfully execute GraphProfFinalize."); | |||||
} | |||||
return ret; | |||||
} | |||||
bool TransProfConfigToParam(const aclgrphProfConfig *profiler_config, vector<string> &prof_config_params) { | |||||
prof_config_params.clear(); | |||||
prof_config_params.emplace_back(kDeviceNums); | |||||
prof_config_params.emplace_back(std::to_string(profiler_config->config.devNums)); | |||||
prof_config_params.emplace_back(kDeviceIdList); | |||||
std::string devID = ""; | |||||
if (profiler_config->config.devNums == 0) { | |||||
GELOGW("The device num is invalid."); | |||||
return false; | |||||
} | |||||
for (uint32_t i = 0; i < profiler_config->config.devNums; i++) { | |||||
devID.append(std::to_string(profiler_config->config.devIdList[i])); | |||||
if (i != profiler_config->config.devNums - 1) { | |||||
devID.append(","); | |||||
} | |||||
} | |||||
prof_config_params.push_back(devID); | |||||
prof_config_params.push_back(kAicoreMetrics); | |||||
auto iter = | |||||
kProfAicoreMetricsToString.find(static_cast<ProfilingAicoreMetrics>(profiler_config->config.aicoreMetrics)); | |||||
if (iter == kProfAicoreMetricsToString.end()) { | |||||
GELOGW("The prof aicore metrics is invalid."); | |||||
return false; | |||||
} | |||||
prof_config_params.push_back(iter->second); | |||||
return true; | |||||
} | |||||
bool isProfConfigValid(const uint32_t *deviceid_list, uint32_t device_nums) { | |||||
if (deviceid_list == nullptr) { | |||||
GELOGE(PARAM_INVALID, "deviceIdList is nullptr"); | |||||
return false; | |||||
} | |||||
if (device_nums == 0 || device_nums > kMaxDeviceNum) { | |||||
GELOGE(PARAM_INVALID, "The device nums is invalid."); | |||||
return false; | |||||
} | |||||
// real device num | |||||
int32_t dev_count = 0; | |||||
rtError_t rt_err = rtGetDeviceCount(&dev_count); | |||||
if (rt_err != RT_ERROR_NONE) { | |||||
GELOGE(INTERNAL_ERROR, "Get the Device count fail."); | |||||
return false; | |||||
} | |||||
if (device_nums > static_cast<uint32_t>(dev_count)) { | |||||
GELOGE(PARAM_INVALID, "Device num(%u) is not in range 1 ~ %d.", device_nums, dev_count); | |||||
return false; | |||||
} | |||||
std::unordered_set<uint32_t> record; | |||||
for (size_t i = 0; i < device_nums; ++i) { | |||||
uint32_t dev_id = deviceid_list[i]; | |||||
if (dev_id >= static_cast<uint32_t>(dev_count)) { | |||||
GELOGE(PARAM_INVALID, "Device id %u is not in range 0 ~ %d(exclude %d)", dev_id, dev_count, dev_count); | |||||
return false; | |||||
} | |||||
if (record.count(dev_id) > 0) { | |||||
GELOGE(PARAM_INVALID, "Device id %u is duplicatedly set", dev_id); | |||||
return false; | |||||
} | |||||
record.insert(dev_id); | |||||
} | |||||
return true; | |||||
} | |||||
aclgrphProfConfig *aclgrphProfCreateConfig(uint32_t *deviceid_list, uint32_t device_nums, | |||||
ProfilingAicoreMetrics aicore_metrics, ProfAicoreEvents *aicore_events, | |||||
uint64_t data_type_config) { | |||||
if (!isProfConfigValid(deviceid_list, device_nums)) { | |||||
return nullptr; | |||||
} | |||||
aclgrphProfConfig *config = new (std::nothrow) aclgrphProfConfig(); | |||||
if (config == nullptr) { | |||||
GELOGE(INTERNAL_ERROR, "new aclgrphProfConfig fail"); | |||||
return nullptr; | |||||
} | |||||
config->config.devNums = device_nums; | |||||
if (memcpy_s(config->config.devIdList, sizeof(config->config.devIdList), deviceid_list, | |||||
device_nums * sizeof(uint32_t)) != EOK) { | |||||
GELOGE(INTERNAL_ERROR, "copy devID failed. size = %u", device_nums); | |||||
delete config; | |||||
return nullptr; | |||||
} | |||||
config->config.aicoreMetrics = static_cast<ProfAicoreMetrics>(aicore_metrics); | |||||
config->config.dataTypeConfig = data_type_config; | |||||
GELOGI("Successfully create prof config."); | |||||
return config; | |||||
} | |||||
Status aclgrphProfDestroyConfig(aclgrphProfConfig *profiler_config) { | |||||
if (profiler_config == nullptr) { | |||||
GELOGE(PARAM_INVALID, "destroy profilerConfig failed, profilerConfig must not be nullptr"); | |||||
return PARAM_INVALID; | |||||
} | |||||
delete profiler_config; | |||||
GELOGI("Successfully destroy prof config."); | |||||
return SUCCESS; | |||||
} | |||||
Status aclgrphProfStart(aclgrphProfConfig *profiler_config) { | |||||
if (profiler_config == nullptr) { | |||||
GELOGE(PARAM_INVALID, "aclgrphProfConfig is invalid."); | |||||
return FAILED; | |||||
} | |||||
std::shared_ptr<GELib> instance_ptr = ge::GELib::GetInstance(); | |||||
if (instance_ptr == nullptr || !instance_ptr->InitFlag()) { | |||||
GELOGE(GE_CLI_GE_NOT_INITIALIZED, "Ge client is not initialized."); | |||||
return FAILED; | |||||
} | |||||
std::lock_guard<std::mutex> lock(g_prof_mutex_); | |||||
// if command mode is set, just return | |||||
if (ProfilingManager::Instance().ProfilingOn()) { | |||||
GELOGW("Graph prof finalize failed, cause profiling command pattern is running."); | |||||
return GE_PROF_MODE_CONFLICT; | |||||
} | |||||
if (!g_graph_prof_init_) { | |||||
GELOGE(GE_PROF_NOT_INIT, "Graph not profiling initialize."); | |||||
return GE_PROF_NOT_INIT; | |||||
} | |||||
Status ret = ProfStartProfiling(&profiler_config->config); | |||||
if (ret != SUCCESS) { | |||||
GELOGE(ret, "Start profiling failed, prof result = %d", ret); | |||||
return FAILED; | |||||
} | |||||
std::vector<string> prof_params; | |||||
if (!TransProfConfigToParam(profiler_config, prof_params)) { | |||||
GELOGE(PARAM_INVALID, "Transfer profilerConfig to string vector failed"); | |||||
return PARAM_INVALID; | |||||
} | |||||
GraphLoader graph_loader; | |||||
Command command; | |||||
command.cmd_params.clear(); | |||||
command.cmd_type = kProfilingStart; | |||||
command.cmd_params = prof_params; | |||||
command.module_index = profiler_config->config.dataTypeConfig; | |||||
GELOGI("Profiling will start, device nums:%s , deviceID:[%s], data type config: 0x%llx", prof_params[0].c_str(), | |||||
prof_params[kDeviceListIndex].c_str(), command.module_index); | |||||
ret = graph_loader.CommandHandle(command); | |||||
if (ret != SUCCESS) { | |||||
GELOGE(ret, "Handle profiling command failed"); | |||||
return FAILED; | |||||
} | |||||
GELOGI("Successfully execute GraphProfStartProfiling."); | |||||
return SUCCESS; | |||||
} | |||||
Status aclgrphProfStop(aclgrphProfConfig *profiler_config) { | |||||
if (profiler_config == nullptr) { | |||||
GELOGE(PARAM_INVALID, "aclgrphProfConfig is invalid."); | |||||
return FAILED; | |||||
} | |||||
std::shared_ptr<GELib> instance_ptr = ge::GELib::GetInstance(); | |||||
if (instance_ptr == nullptr || !instance_ptr->InitFlag()) { | |||||
GELOGE(GE_CLI_GE_NOT_INITIALIZED, "Ge client is not initialized."); | |||||
return FAILED; | |||||
} | |||||
std::lock_guard<std::mutex> lock(g_prof_mutex_); | |||||
// if command mode is set, just return | |||||
if (ProfilingManager::Instance().ProfilingOn()) { | |||||
GELOGW("Graph prof finalize failed, cause profiling command pattern is running."); | |||||
return GE_PROF_MODE_CONFLICT; | |||||
} | |||||
if (!g_graph_prof_init_) { | |||||
GELOGE(GE_PROF_NOT_INIT, "Graph not profiling initialize."); | |||||
return GE_PROF_NOT_INIT; | |||||
} | |||||
for (uint32_t i = 0; i < profiler_config->config.devNums; i++) { | |||||
uint64_t data_type_config; | |||||
Status status = ProfGetDataTypeConfig(profiler_config->config.devIdList[i], data_type_config); | |||||
if (status != SUCCESS) { | |||||
GELOGE(status, "Prof get data type config failed, prof result = %d", status); | |||||
return status; | |||||
} | |||||
if (data_type_config != profiler_config->config.dataTypeConfig) { | |||||
GELOGE(FAILED, "data type config verify failed"); | |||||
return FAILED; | |||||
} | |||||
} | |||||
std::vector<string> prof_params; | |||||
if (!TransProfConfigToParam(profiler_config, prof_params)) { | |||||
GELOGE(PARAM_INVALID, "Transfer profilerConfig to string vector failed"); | |||||
return PARAM_INVALID; | |||||
} | |||||
GraphLoader graph_loader; | |||||
Command command; | |||||
command.cmd_params.clear(); | |||||
command.cmd_type = kProfilingStop; | |||||
command.cmd_params = prof_params; | |||||
command.module_index = profiler_config->config.dataTypeConfig; | |||||
GELOGI("Profiling will stop, device nums:%s , deviceID:[%s], data type config: 0x%llx", prof_params[0].c_str(), | |||||
prof_params[kDeviceListIndex].c_str(), command.module_index); | |||||
Status ret = graph_loader.CommandHandle(command); | |||||
if (ret != SUCCESS) { | |||||
GELOGE(ret, "Handle profiling command failed"); | |||||
return FAILED; | |||||
} | |||||
ret = ProfStopProfiling(&profiler_config->config); | |||||
if (ret != SUCCESS) { | |||||
GELOGE(ret, "Stop profiling failed, prof result = %d", ret); | |||||
return ret; | |||||
} | |||||
GELOGI("Successfully execute GraphProfStopProfiling."); | |||||
return SUCCESS; | |||||
} | |||||
} // namespace ge |
@@ -4,7 +4,6 @@ LOCAL_PATH := $(call my-dir) | |||||
COMMON_LOCAL_SRC_FILES := \ | COMMON_LOCAL_SRC_FILES := \ | ||||
proto/ge_api.proto \ | proto/ge_api.proto \ | ||||
ge_api.cc \ | ge_api.cc \ | ||||
ge_prof.cc \ | |||||
COMMON_LOCAL_C_INCLUDES := \ | COMMON_LOCAL_C_INCLUDES := \ | ||||
@@ -69,9 +68,9 @@ LOCAL_SHARED_LIBRARIES := \ | |||||
libgraph \ | libgraph \ | ||||
libregister \ | libregister \ | ||||
libge_compiler \ | libge_compiler \ | ||||
libge_common \ | |||||
libmsprof | |||||
libge_common | |||||
LOCAL_STATIC_LIBRARIES += libmsprofiler_fwk \ | |||||
LOCAL_LDFLAGS := -lrt -ldl | LOCAL_LDFLAGS := -lrt -ldl | ||||
@@ -104,8 +103,10 @@ LOCAL_SHARED_LIBRARIES := \ | |||||
libregister \ | libregister \ | ||||
libruntime \ | libruntime \ | ||||
libge_compiler \ | libge_compiler \ | ||||
libge_common \ | |||||
libmsprof | |||||
libge_common | |||||
LOCAL_STATIC_LIBRARIES += libmsprofiler_fwk \ | |||||
LOCAL_LDFLAGS := -lrt -ldl | LOCAL_LDFLAGS := -lrt -ldl | ||||
@@ -0,0 +1,199 @@ | |||||
/** | |||||
* Copyright 2020 Huawei Technologies Co., Ltd | |||||
* | |||||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||||
* you may not use this file except in compliance with the License. | |||||
* You may obtain a copy of the License at | |||||
* | |||||
* http://www.apache.org/licenses/LICENSE-2.0 | |||||
* | |||||
* Unless required by applicable law or agreed to in writing, software | |||||
* distributed under the License is distributed on an "AS IS" BASIS, | |||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
* See the License for the specific language governing permissions and | |||||
* limitations under the License. | |||||
*/ | |||||
#include "common/profiling/ge_profiling.h" | |||||
#include "runtime/base.h" | |||||
#include "common/profiling/profiling_manager.h" | |||||
#include "framework/common/debug/ge_log.h" | |||||
#include "framework/common/debug/log.h" | |||||
#include "graph/load/graph_loader.h" | |||||
#include "init/gelib.h" | |||||
#include "framework/common/ge_inner_error_codes.h" | |||||
namespace { | |||||
const uint32_t kDeviceListIndex = 3; | |||||
const std::string kDeviceNums = "devNums"; | |||||
const std::string kDeviceIdList = "devIdList"; | |||||
const std::string kProfilingInit = "prof_init"; | |||||
const std::string kProfilingFinalize = "prof_finalize"; | |||||
const std::string kProfilingStart = "prof_start"; | |||||
const std::string kProfilingStop = "prof_stop"; | |||||
const std::string kProfModelSubscribe = "prof_model_subscribe"; | |||||
const std::string kProfModelUnsubscribe = "prof_model_cancel_subscribe"; | |||||
const std::string kRtSetDeviceRegName = "profiling"; | |||||
const std::map<ProfCommandHandleType, std::string> kProfCommandTypeMap = { | |||||
{kProfCommandhandleInit, kProfilingInit}, | |||||
{kProfCommandhandleStart, kProfilingStart}, | |||||
{kProfCommandhandleStop, kProfilingStop}, | |||||
{kProfCommandhandleFinalize, kProfilingFinalize}, | |||||
{kProfCommandhandleModelSubscribe, kProfModelSubscribe}, | |||||
{kProfCommandhandleModelUnsubscribe, kProfModelUnsubscribe}}; | |||||
} // namespace | |||||
bool TransProfConfigToParam(const ProfCommandHandleData &profCommand, vector<string> &prof_config_params) { | |||||
prof_config_params.clear(); | |||||
prof_config_params.emplace_back(kDeviceNums); | |||||
prof_config_params.emplace_back(std::to_string(profCommand.devNums)); | |||||
prof_config_params.emplace_back(kDeviceIdList); | |||||
std::string devID = ""; | |||||
if (profCommand.devNums == 0) { | |||||
GELOGW("The device num is invalid."); | |||||
return false; | |||||
} | |||||
for (uint32_t i = 0; i < profCommand.devNums; i++) { | |||||
devID.append(std::to_string(profCommand.devIdList[i])); | |||||
if (i != profCommand.devNums - 1) { | |||||
devID.append(","); | |||||
} | |||||
} | |||||
prof_config_params.push_back(devID); | |||||
return true; | |||||
} | |||||
bool isProfConfigValid(const uint32_t *deviceid_list, uint32_t device_nums) { | |||||
if (deviceid_list == nullptr) { | |||||
GELOGE(ge::PARAM_INVALID, "deviceIdList is nullptr"); | |||||
return false; | |||||
} | |||||
if (device_nums == 0 || device_nums > MAX_DEV_NUM) { | |||||
GELOGE(ge::PARAM_INVALID, "The device nums: %u is invalid.", device_nums); | |||||
return false; | |||||
} | |||||
// real device num | |||||
int32_t dev_count = 0; | |||||
rtError_t rt_err = rtGetDeviceCount(&dev_count); | |||||
if (rt_err != RT_ERROR_NONE) { | |||||
GELOGE(ge::INTERNAL_ERROR, "Get the Device count fail."); | |||||
return false; | |||||
} | |||||
if (device_nums > static_cast<uint32_t>(dev_count)) { | |||||
GELOGE(ge::PARAM_INVALID, "Device num(%u) is not in range 1 ~ %d.", device_nums, dev_count); | |||||
return false; | |||||
} | |||||
std::unordered_set<uint32_t> record; | |||||
for (size_t i = 0; i < device_nums; ++i) { | |||||
uint32_t dev_id = deviceid_list[i]; | |||||
if (dev_id >= static_cast<uint32_t>(dev_count)) { | |||||
GELOGE(ge::PARAM_INVALID, "Device id %u is not in range 0 ~ %d(exclude %d)", dev_id, dev_count, dev_count); | |||||
return false; | |||||
} | |||||
if (record.count(dev_id) > 0) { | |||||
GELOGE(ge::PARAM_INVALID, "Device id %u is duplicatedly set", dev_id); | |||||
return false; | |||||
} | |||||
record.insert(dev_id); | |||||
} | |||||
return true; | |||||
} | |||||
ge::Status RegProfCtrlCallback(MsprofCtrlCallback func) { | |||||
if (func == nullptr) { | |||||
GELOGE(ge::PARAM_INVALID, "Msprof ctrl callback is nullptr."); | |||||
return ge::PARAM_INVALID; | |||||
} | |||||
if (ge::ProfilingManager::Instance().GetMsprofCallback().msprofCtrlCallback != nullptr) { | |||||
GELOGW("Msprof ctrl callback is exist, just ignore it."); | |||||
} else { | |||||
GELOGI("GE register Msprof ctrl callback."); | |||||
ge::ProfilingManager::Instance().SetMsprofCtrlCallback(func); | |||||
} | |||||
return ge::SUCCESS; | |||||
} | |||||
ge::Status RegProfSetDeviceCallback(MsprofSetDeviceCallback func) { | |||||
if (func == nullptr) { | |||||
GELOGE(ge::PARAM_INVALID, "MsprofSetDeviceCallback callback is nullptr."); | |||||
return ge::PARAM_INVALID; | |||||
} | |||||
// Pass MsprofSetDeviceCallback to runtime | |||||
GELOGI("GE pass setdevice callback to runtime."); | |||||
ge::Status rt_ret = rtRegDeviceStateCallback(kRtSetDeviceRegName.c_str(), static_cast<rtDeviceStateCallback>(func)); | |||||
if (rt_ret != ge::SUCCESS) { | |||||
GELOGE(rt_ret, "Pass MsprofSetDeviceCallback to runtime failed!"); | |||||
return rt_ret; | |||||
} | |||||
return ge::SUCCESS; | |||||
} | |||||
ge::Status RegProfReporterCallback(MsprofReporterCallback func) { | |||||
if (func == nullptr) { | |||||
GELOGE(ge::PARAM_INVALID, "MsprofReporterCallback callback is nullptr."); | |||||
return ge::PARAM_INVALID; | |||||
} | |||||
if (ge::ProfilingManager::Instance().GetMsprofCallback().msprofReporterCallback != nullptr) { | |||||
GELOGW("Msprof reporter callback is exist, just ignore it."); | |||||
} else { | |||||
GELOGI("GE register Msprof reporter callback."); | |||||
ge::ProfilingManager::Instance().SetMsprofReporterCallback(func); | |||||
// Pass MsprofReporterCallback to runtime | |||||
ge::Status rt_ret = rtSetMsprofReporterCallback(func); | |||||
if (rt_ret != ge::SUCCESS) { | |||||
GELOGE(rt_ret, "Pass MsprofReporterCallback to runtime failed!!"); | |||||
return rt_ret; | |||||
} | |||||
// Pass MsprofReporterCallback to hccl | |||||
} | |||||
return ge::SUCCESS; | |||||
} | |||||
ge::Status ProfCommandHandle(ProfCommandHandleType type, void *data, uint32_t len) { | |||||
if (type != kProfCommandhandleFinalize) { | |||||
GE_CHECK_NOTNULL(data); | |||||
} | |||||
ProfCommandHandleData *prof_config_param = (ProfCommandHandleData *)data; | |||||
auto iter = kProfCommandTypeMap.find(type); | |||||
if (iter == kProfCommandTypeMap.end()) { | |||||
GELOGW("The prof comand type is invalid."); | |||||
return ge::PARAM_INVALID; | |||||
} | |||||
std::vector<string> prof_params; | |||||
if (type == kProfCommandhandleStart || type == kProfCommandhandleStop) { | |||||
if (!isProfConfigValid(prof_config_param->devIdList, prof_config_param->devNums)) { | |||||
return ge::FAILED; | |||||
} | |||||
if (!TransProfConfigToParam(*prof_config_param, prof_params)) { | |||||
GELOGE(ge::PARAM_INVALID, "Transfer profilerConfig to string vector failed"); | |||||
return ge::PARAM_INVALID; | |||||
} | |||||
} | |||||
ge::GraphLoader graph_loader; | |||||
ge::Command command; | |||||
command.cmd_params.clear(); | |||||
command.cmd_type = iter->second; | |||||
command.cmd_params = prof_params; | |||||
if (type != kProfCommandhandleFinalize) { | |||||
command.module_index = prof_config_param->profSwitch; | |||||
} | |||||
GELOGI("GE commandhandle execute, Command Type: %d, data type config: 0x%llx", type, command.module_index); | |||||
if (type == kProfCommandhandleStart || type == kProfCommandhandleStop) { | |||||
GELOGI("Profiling device nums:%s , deviceID:[%s]", prof_params[0].c_str(), prof_params[kDeviceListIndex].c_str()); | |||||
} | |||||
ge::Status ret = graph_loader.CommandHandle(command); | |||||
if (ret != ge::SUCCESS) { | |||||
GELOGE(ret, "Handle profiling command failed"); | |||||
return ge::FAILED; | |||||
} | |||||
GELOGI("Successfully execute profiling command type: %d, command 0x%llx.", type, command.module_index); | |||||
return ge::SUCCESS; | |||||
} | |||||
@@ -0,0 +1,26 @@ | |||||
/** | |||||
* Copyright 2020 Huawei Technologies Co., Ltd | |||||
* | |||||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||||
* you may not use this file except in compliance with the License. | |||||
* You may obtain a copy of the License at | |||||
* | |||||
* http://www.apache.org/licenses/LICENSE-2.0 | |||||
* | |||||
* Unless required by applicable law or agreed to in writing, software | |||||
* distributed under the License is distributed on an "AS IS" BASIS, | |||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
* See the License for the specific language governing permissions and | |||||
* limitations under the License. | |||||
*/ | |||||
#include "common/profiling/ge_runner_profiling.h" | |||||
#include "init/gelib.h" | |||||
bool IsInitialize() { | |||||
std::shared_ptr<ge::GELib> instance_ptr = ge::GELib::GetInstance(); | |||||
if (instance_ptr == nullptr || instance_ptr->InitFlag() == false) { | |||||
return false; | |||||
} | |||||
return true; | |||||
} |
@@ -24,16 +24,9 @@ | |||||
#include "graph/load/new_model_manager/davinci_model.h" | #include "graph/load/new_model_manager/davinci_model.h" | ||||
namespace { | namespace { | ||||
const char *const kJobID = "jobID"; | |||||
const char *const kDeviceID = "deviceID"; | |||||
const char *const kStartCfg = "startCfg"; | |||||
const char *const kFeatures = "features"; | |||||
const char *const kConf = "conf"; | |||||
const char *const kEvents = "events"; | |||||
const char *const kAiCoreEvents = "ai_core_events"; | |||||
const char *const kName = "name"; | |||||
const char *const kTraceID = "traceId"; | |||||
const char *const kProfDir = "resultPath"; | |||||
const char *const kTrainingTrace = "training_trace"; | |||||
const char *const kFpPoint = "fp_point"; | |||||
const char *const kBpPoint = "bp_point"; | |||||
const size_t kReportMaxLen = 2048; | const size_t kReportMaxLen = 2048; | ||||
const int32_t kMaxDeviceNum = 256; | const int32_t kMaxDeviceNum = 256; | ||||
const std::string kConfigNumsdev = "devNums"; | const std::string kConfigNumsdev = "devNums"; | ||||
@@ -45,7 +38,13 @@ const std::string kProfModelUnsubscribe = "prof_model_cancel_subscribe"; | |||||
} // namespace | } // namespace | ||||
namespace ge { | namespace ge { | ||||
ProfilingManager::ProfilingManager() : subscribe_count_(0) {} | |||||
ProfilingManager::ProfilingManager() : is_load_profiling_(false), | |||||
is_execute_profiling_(false), | |||||
is_training_trace_(false), | |||||
subscribe_count_(0) { | |||||
prof_cb_.msprofCtrlCallback = nullptr; | |||||
prof_cb_.msprofReporterCallback = nullptr; | |||||
} | |||||
ProfilingManager::~ProfilingManager() {} | ProfilingManager::~ProfilingManager() {} | ||||
@@ -58,44 +57,29 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ge::Status ProfilingManager::In | |||||
#ifdef DAVINCI_SUPPORT_PROFILING | #ifdef DAVINCI_SUPPORT_PROFILING | ||||
vector<int32_t>().swap(device_id_); | vector<int32_t>().swap(device_id_); | ||||
subscribe_count_ = 0; | subscribe_count_ = 0; | ||||
job_id_ = options.job_id; | |||||
GELOGI("ProfilingManager::Init job_id:%s", job_id_.c_str()); | |||||
GELOGI("ProfilingManager::Init job_id:%s", options.job_id.c_str()); | |||||
Status ret; | |||||
if (!recv_profiling_config_.empty()) { | |||||
GELOGI("Profiling json config from acl:%s", recv_profiling_config_.c_str()); | |||||
ret = InitFromAclCfg(recv_profiling_config_); | |||||
} else { | |||||
ret = InitFromOptions(options); | |||||
if (ret == SUCCESS && is_load_profiling_) { | |||||
device_id_.push_back(options.device_id); | |||||
} | |||||
} | |||||
struct MsprofGeOptions prof_conf = {{ 0 }}; | |||||
Status ret = InitFromOptions(options, prof_conf); | |||||
if (ret != SUCCESS) { | if (ret != SUCCESS) { | ||||
GELOGE(ret, "Failed to init profiling."); | GELOGE(ret, "Failed to init profiling."); | ||||
return ret; | return ret; | ||||
} | } | ||||
if (is_load_profiling_) { | |||||
// register Framework to profiling | |||||
int result = Msprof::Engine::Init(GE_PROFILING_MODULE, &engine_); | |||||
if (result != 0) { | |||||
GELOGE(FAILED, "Register profiling engine failed."); | |||||
return FAILED; | |||||
if (is_execute_profiling_) { | |||||
if (prof_cb_.msprofCtrlCallback == nullptr) { | |||||
GELOGE(ge::PARAM_INVALID, "MsprofCtrlCallback callback is nullptr."); | |||||
return ge::PARAM_INVALID; | |||||
} | } | ||||
// profiling startup first time | |||||
GELOGI("Begin to init profiling, device num %zu", device_id_.size()); | |||||
for (size_t i = 0; i < device_id_.size(); ++i) { | |||||
ret = StartProfiling(0, device_id_[i]); | |||||
if (ret != SUCCESS) { | |||||
GELOGW("Profiling start failed on device %d.", device_id_[i]); | |||||
continue; | |||||
} | |||||
GELOGI("Profiling init succ on device %d.", device_id_[i]); | |||||
int32_t cb_ret = prof_cb_.msprofCtrlCallback( | |||||
static_cast<uint32_t>(MsprofCtrlCallbackType::MSPROF_CTRL_INIT_GE_OPTIONS), | |||||
static_cast<void *>(&prof_conf), sizeof(MsprofGeOptions)); | |||||
if (cb_ret != 0) { | |||||
GELOGE(FAILED, "Call msprofCtrlCallback failed, type:%u, return:%d", | |||||
static_cast<uint32_t>(MsprofCtrlCallbackType::MSPROF_CTRL_INIT_GE_OPTIONS), cb_ret); | |||||
return FAILED; | |||||
} | } | ||||
GELOGI("Profiling init success"); | |||||
} else { | } else { | ||||
GELOGI("The profiling is off, skip the initialization"); | GELOGI("The profiling is off, skip the initialization"); | ||||
} | } | ||||
@@ -103,288 +87,116 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ge::Status ProfilingManager::In | |||||
return SUCCESS; | return SUCCESS; | ||||
} | } | ||||
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ge::Status ProfilingManager::InitFromAclCfg( | |||||
const std::string &config) { | |||||
ge::Status ProfilingManager::InitFromOptions(const Options &options, MsprofGeOptions &prof_conf) { | |||||
#ifdef DAVINCI_SUPPORT_PROFILING | #ifdef DAVINCI_SUPPORT_PROFILING | ||||
try { | |||||
is_load_profiling_ = false; | |||||
is_execute_profiling_ = false; | |||||
profiling_opts_.clear(); | |||||
op_trace_conf_.clear(); | |||||
Json start_prof_conf = Json::parse(config); | |||||
Json &prof_conf = start_prof_conf[kStartCfg][0]; | |||||
job_id_ = prof_conf[kJobID]; | |||||
auto iter = prof_conf.find(kProfDir); | |||||
if (iter != prof_conf.end()) { | |||||
prof_dir_ = prof_conf[kProfDir]; | |||||
} | |||||
Json &device_id = prof_conf[kDeviceID]; | |||||
if (device_id.size() != 0) { | |||||
vector<int32_t>().swap(device_id_); | |||||
bool is_all = false; | |||||
for (size_t i = 0; i < device_id.size(); i++) { | |||||
std::string device_id_str = device_id[i].get<std::string>(); | |||||
if (device_id_str == "all") { | |||||
is_all = true; | |||||
break; | |||||
} | |||||
device_id_.push_back(std::stoi(device_id_str)); | |||||
} | |||||
if (is_all) { | |||||
int32_t count = 0; | |||||
rtError_t rt_err = rtGetDeviceCount(&count); | |||||
if (rt_err != RT_ERROR_NONE) { | |||||
GELOGE(FAILED, "Call rtGetDeviceCount to get device failed."); | |||||
} | |||||
vector<int32_t>().swap(device_id_); | |||||
for (int32_t i = 0; i < count; ++i) { | |||||
device_id_.push_back(i); | |||||
} | |||||
} | |||||
// enable profiling by env | |||||
char env_profiling_mode[MMPA_MAX_PATH] = { 0x00 }; | |||||
is_load_profiling_ = false; // Change in ProfInit | |||||
is_execute_profiling_ = false; | |||||
if (options.profiling_mode == "1" && !options.profiling_options.empty()) { | |||||
// enable profiling by ge option | |||||
if (memcpy_s(prof_conf.options, MSPROF_OPTIONS_DEF_LEN_MAX, options.profiling_options.c_str(), | |||||
options.profiling_options.size()) != EOK) { | |||||
GELOGE(INTERNAL_ERROR, "copy profiling_options failed."); | |||||
return INTERNAL_ERROR; | |||||
} | } | ||||
Json &features = prof_conf[kFeatures]; | |||||
if (ParseFeaturesFromAclCfg(features) != SUCCESS) { | |||||
GELOGE(FAILED, "Parse feature from acl cfg failed."); | |||||
return FAILED; | |||||
is_execute_profiling_ = true; | |||||
GELOGI("The profiling in options is %s, %s. origin option: %s", options.profiling_mode.c_str(), | |||||
prof_conf.options, options.profiling_options.c_str()); | |||||
} else { | |||||
(void)mmGetEnv("PROFILING_MODE", env_profiling_mode, MMPA_MAX_PATH); | |||||
(void)mmGetEnv("PROFILING_OPTIONS", prof_conf.options, MSPROF_OPTIONS_DEF_LEN_MAX); | |||||
// The env is invalid | |||||
if ((strcmp("true", env_profiling_mode) != 0) || (strcmp(prof_conf.options, "\0") == 0)) { | |||||
return SUCCESS; | |||||
} | } | ||||
is_load_profiling_ = true; | |||||
// enable profiling by env | |||||
is_execute_profiling_ = true; | is_execute_profiling_ = true; | ||||
} catch (...) { | |||||
GELOGE(FAILED, "Json conf is not invalid !"); | |||||
GELOGI("The profiling in env is %s, %s", env_profiling_mode, prof_conf.options); | |||||
} | |||||
if (!is_execute_profiling_) { | |||||
return SUCCESS; | |||||
} | |||||
// Parse json str for bp fp | |||||
Status ret = ParseOptions(prof_conf.options); | |||||
if (ret != ge::SUCCESS) { | |||||
GELOGE(ge::PARAM_INVALID, "Parse training trace param failed."); | |||||
return ge::PARAM_INVALID; | return ge::PARAM_INVALID; | ||||
} | } | ||||
if (memcpy_s(prof_conf.jobId, sizeof(prof_conf.jobId), options.job_id.c_str(), | |||||
sizeof(options.job_id.c_str())) != EOK) { | |||||
GELOGE(INTERNAL_ERROR, "copy job_id failed."); | |||||
return INTERNAL_ERROR; | |||||
} | |||||
#endif | #endif | ||||
return ge::SUCCESS; | return ge::SUCCESS; | ||||
} | } | ||||
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ge::Status ProfilingManager::ParseFeaturesFromAclCfg( | |||||
const Json &features) { | |||||
#ifdef DAVINCI_SUPPORT_PROFILING | |||||
ge::Status ProfilingManager::ParseOptions(const std::string &options) { | |||||
if (options.empty()) { | |||||
GELOGE(ge::PARAM_INVALID, "Profiling options is empty."); | |||||
return ge::PARAM_INVALID; | |||||
} | |||||
try { | try { | ||||
for (size_t i = 0; i < features.size(); ++i) { | |||||
const Json &feature = features[i]; | |||||
if ((feature.find(kName) == feature.end()) || feature[kName].is_null()) { | |||||
continue; | |||||
} | |||||
const std::string &name = feature[kName]; | |||||
if (name == "op_trace") { | |||||
const Json &conf = feature[kConf]; | |||||
const Json &events = conf[0][kEvents]; | |||||
const std::string &ai_core_events = events[0][kAiCoreEvents]; | |||||
GELOGI("Op trace config from acl ai_core_events:%s", ai_core_events.c_str()); | |||||
is_op_trace_ = true; | |||||
ProfMgrConf prof_mgr_conf; | |||||
int result = ProfMgrGetConf(ai_core_events, &prof_mgr_conf); | |||||
if (result != 0) { | |||||
GELOGE(FAILED, "ProfMgrGetConf failed."); | |||||
return FAILED; | |||||
} | |||||
op_trace_conf_ = prof_mgr_conf.conf; | |||||
op_trace_iter_num_ = static_cast<int32_t>(op_trace_conf_.size()); | |||||
GELOGI("Op trace profiling iter num %d,", op_trace_iter_num_); | |||||
} else if (name == "task_trace") { | |||||
is_op_trace_ = false; | |||||
if (feature.find(kConf) != feature.end()) { | |||||
const Json &conf = feature[kConf]; | |||||
std::stringstream task_trace_conf; | |||||
task_trace_conf << conf; | |||||
task_trace_conf_ = task_trace_conf.str(); | |||||
} | |||||
GELOGI("Task trace config from acl"); | |||||
} else if (name == "system_trace") { | |||||
is_op_trace_ = false; | |||||
const Json &conf = feature[kConf]; | |||||
std::stringstream system_trace_conf; | |||||
system_trace_conf << conf; | |||||
system_trace_conf_ = system_trace_conf.str(); | |||||
GELOGI("System trace config from acl"); | |||||
} | |||||
profiling_opts_.push_back(name); | |||||
Json prof_options = Json::parse(options); | |||||
const std::string training_trace = prof_options[kTrainingTrace]; | |||||
if (training_trace.empty()) { | |||||
GELOGI("Training trace will not take effect."); | |||||
return ge::SUCCESS; | |||||
} | |||||
GELOGI("GE profiling training trace:%s", training_trace.c_str()); | |||||
if (training_trace != "on") { | |||||
GELOGE(ge::PARAM_INVALID, "Training trace param:%s is invalid.", training_trace.c_str()); | |||||
return ge::PARAM_INVALID; | |||||
} | |||||
fp_point_ = prof_options[kFpPoint]; | |||||
bp_point_ = prof_options[kBpPoint]; | |||||
if (!fp_point_.empty() && !bp_point_.empty()) { | |||||
GELOGI("Training trace bp fp is set, bp_point:%s, fp_point:%s.", bp_point_.c_str(), fp_point_.c_str()); | |||||
} | } | ||||
} catch (...) { | } catch (...) { | ||||
GELOGE(ge::PARAM_INVALID, "Json conf feature is not invalid !"); | |||||
GELOGE(FAILED, "Json prof_conf options is invalid."); | |||||
return ge::PARAM_INVALID; | return ge::PARAM_INVALID; | ||||
} | } | ||||
#endif | |||||
return ge::SUCCESS; | return ge::SUCCESS; | ||||
} | } | ||||
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ge::Status ProfilingManager::InitFromOptions(const Options &options) { | |||||
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::StopProfiling() { | |||||
#ifdef DAVINCI_SUPPORT_PROFILING | #ifdef DAVINCI_SUPPORT_PROFILING | ||||
// enable profiling support two ways: env and front end | |||||
char profiling_mode_temp[MMPA_MAX_PATH] = { 0x00 }; | |||||
char prof_options_temp[MMPA_MAX_PATH] = { 0x00 }; | |||||
(void)mmGetEnv("PROFILING_MODE", profiling_mode_temp, MMPA_MAX_PATH); | |||||
(void)mmGetEnv("PROFILING_OPTIONS", prof_options_temp, MMPA_MAX_PATH); | |||||
const char *profiling_mode = profiling_mode_temp; | |||||
const char *prof_options = prof_options_temp; | |||||
if ((profiling_mode == nullptr) || (strcmp("true", profiling_mode) != 0) || (prof_options == nullptr)) { | |||||
is_load_profiling_ = false; | |||||
is_execute_profiling_ = false; | |||||
} else { | |||||
std::string prof_options_str = std::string(prof_options); | |||||
profiling_opts_ = StringUtils::Split(prof_options_str, ':'); | |||||
is_load_profiling_ = true; | |||||
is_execute_profiling_ = true; | |||||
GELOGI("The profiling in env is %s, %s", profiling_mode, prof_options); | |||||
} | |||||
if (!is_load_profiling_) { | |||||
const std::string enable_profiling = "1"; | |||||
if (options.profiling_mode != enable_profiling || options.profiling_options.empty()) { | |||||
is_load_profiling_ = false; | |||||
is_execute_profiling_ = false; | |||||
return SUCCESS; | |||||
} else { | |||||
profiling_opts_ = StringUtils::Split(options.profiling_options, ':'); | |||||
is_load_profiling_ = true; | |||||
is_execute_profiling_ = true; | |||||
GELOGI("The profiling in options is %s, %s", options.profiling_mode.c_str(), options.profiling_options.c_str()); | |||||
} | |||||
} | |||||
// features:'training_trace', 'task_trace' or 'op_trace' etc | |||||
if (!profiling_opts_.empty()) { | |||||
if (profiling_opts_[0] == "op_trace") { | |||||
is_op_trace_ = true; | |||||
// op trace get conf | |||||
ProfMgrConf prof_mgr_conf; | |||||
int result = ProfMgrGetConf("", &prof_mgr_conf); | |||||
if (result != 0) { | |||||
GELOGE(FAILED, "ProfMgrGetConf failed."); | |||||
return FAILED; | |||||
} | |||||
op_trace_conf_ = prof_mgr_conf.conf; | |||||
op_trace_iter_num_ = static_cast<int32_t>(op_trace_conf_.size()); | |||||
GELOGI("op trace profiling iter num %d,", op_trace_iter_num_); | |||||
} else { | |||||
is_op_trace_ = false; | |||||
op_trace_iter_num_ = 1; | |||||
uint64_t module = GetProfilingModule(); | |||||
// The following if case will not be executed in normal case, inc case of ProfStopProfiling is abnormal | |||||
int32_t device_num = static_cast<int32_t>(device_id_.size()); | |||||
if (device_num != 0) { | |||||
auto device_id_ptr = std::unique_ptr<uint32_t[]>(new (std::nothrow) uint32_t[device_num]); | |||||
if (device_id_ptr == nullptr) { | |||||
GELOGE(FAILED, "Stop profiling: device id ptr is null."); | |||||
return; | |||||
} | } | ||||
} | |||||
#endif | |||||
return ge::SUCCESS; | |||||
} | |||||
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ge::Status ProfilingManager::StartProfiling(int32_t iter_num, | |||||
int32_t device_id) { | |||||
#ifdef DAVINCI_SUPPORT_PROFILING | |||||
if (!profiling_opts_.empty()) { | |||||
GELOGI("Start profiling index is %d", iter_num); | |||||
// current one docker only use one device | |||||
Json p_device; | |||||
try { | |||||
// profiling need physical_device_id | |||||
p_device[kDeviceID] = std::to_string(device_id); | |||||
p_device[kJobID] = job_id_; | |||||
p_device[kTraceID] = std::to_string(GetContext().TraceId()); | |||||
if (!prof_dir_.empty()) { | |||||
p_device[kProfDir] = prof_dir_; | |||||
GELOGI("Prof dir: %s.", prof_dir_.c_str()); | |||||
} | |||||
Json features; | |||||
if (is_op_trace_) { | |||||
Json f; | |||||
f[kName] = "op_trace"; | |||||
Json conf; | |||||
if (op_trace_conf_.size() <= static_cast<size_t>(iter_num)) { | |||||
GELOGE(FAILED, "Op trace iter num is invalid!"); | |||||
return FAILED; | |||||
} | |||||
Json events; | |||||
events[0] = nlohmann::json::parse(op_trace_conf_[iter_num]); | |||||
conf[0][kEvents] = events; | |||||
f[kConf] = conf; | |||||
features[0] = f; | |||||
if (iter_num == 0) { | |||||
is_load_ = true; | |||||
} | |||||
} else { | |||||
for (std::vector<std::string>::size_type i = 0; i < profiling_opts_.size(); i++) { | |||||
Json f; | |||||
if (profiling_opts_[i] == "system_trace") { | |||||
f[kConf] = nlohmann::json::parse(system_trace_conf_); | |||||
} else if (profiling_opts_[i] == "task_trace") { | |||||
if (!task_trace_conf_.empty()) { | |||||
f[kConf] = nlohmann::json::parse(task_trace_conf_); | |||||
} | |||||
} | |||||
f[kName] = profiling_opts_[i]; | |||||
features[i] = f; | |||||
} | |||||
is_load_ = true; | |||||
} | |||||
p_device[kFeatures] = features; | |||||
// only one device, but sProfMgrStartUp API require for device list | |||||
Json devices; | |||||
devices[0] = p_device; | |||||
Json start_cfg; | |||||
start_cfg[kStartCfg] = devices; | |||||
// convert json to string | |||||
std::stringstream ss; | |||||
ss << start_cfg; | |||||
send_profiling_config_ = ss.str(); | |||||
GELOGI("Profiling config %s\n", send_profiling_config_.c_str()); | |||||
} catch (...) { | |||||
GELOGE(FAILED, "Op trace json conf is not invalid !"); | |||||
return FAILED; | |||||
for (int32_t i = 0; i < device_num; i++) { | |||||
device_id_ptr[i] = static_cast<uint32_t>(device_id_[i]); | |||||
} | } | ||||
// runtime startup for profiling | |||||
uint64_t module = GetProfilingModule(); | |||||
int32_t device_num = 1; | |||||
uint32_t device_id_rt = static_cast<uint32_t>(device_id); | |||||
GE_CHK_RT_RET(rtProfilerStart(module, device_num, &device_id_rt)); | |||||
// call profiling startup API | |||||
ProfMgrCfg prof_cfg = {send_profiling_config_}; | |||||
void *prof_handle = ProfMgrStartUp(&prof_cfg); | |||||
if (prof_handle == nullptr) { | |||||
GELOGW("ProfMgrStartUp failed on device %d ", device_id); | |||||
return FAILED; | |||||
rtError_t rt_ret = rtProfilerStop(module, device_num, device_id_ptr.get()); | |||||
if (rt_ret != RT_ERROR_NONE) { | |||||
GELOGW("Call rtProfilerStop failed, ret:%d", rt_ret); | |||||
} | } | ||||
GELOGD("StartProfiling, prof_handle: %p", prof_handle); | |||||
prof_handle_vec_.push_back(prof_handle); | |||||
} | } | ||||
#endif | |||||
return SUCCESS; | |||||
} | |||||
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::StopProfiling() { | |||||
#ifdef DAVINCI_SUPPORT_PROFILING | |||||
Msprof::Engine::Reporter *reporter = PluginImpl::GetPluginReporter(); | |||||
if (reporter != nullptr) { | |||||
int ret = reporter->Flush(); | |||||
GELOGI("Report data end, ret is %d", ret); | |||||
// stop profiling | |||||
if (prof_cb_.msprofCtrlCallback == nullptr) { | |||||
GELOGE(ge::PARAM_INVALID, "MsprofCtrlCallback callback is nullptr."); | |||||
return; | |||||
} | } | ||||
uint64_t module = GetProfilingModule(); | |||||
int32_t device_num = static_cast<int32_t>(device_id_.size()); | |||||
auto device_id_ptr = std::unique_ptr<uint32_t[]>(new (std::nothrow) uint32_t[device_num]); | |||||
if (device_id_ptr == nullptr) { | |||||
GELOGE(FAILED, "Stop profiling: device id ptr is null."); | |||||
int32_t cb_ret = prof_cb_.msprofCtrlCallback(static_cast<uint32_t>(MsprofCtrlCallbackType::MSPROF_CTRL_FINALIZE), | |||||
nullptr, 0); | |||||
if (cb_ret != 0) { | |||||
GELOGW("call msprofCtrlCallback failed, type:%u, return:%d", | |||||
static_cast<uint32_t>(MsprofCtrlCallbackType::MSPROF_CTRL_FINALIZE), cb_ret); | |||||
return; | return; | ||||
} | } | ||||
for (int32_t i = 0; i < device_num; i++) { | |||||
device_id_ptr[i] = static_cast<uint32_t>(device_id_[i]); | |||||
} | |||||
rtError_t rt_ret = rtProfilerStop(module, device_num, device_id_ptr.get()); | |||||
if (rt_ret != RT_ERROR_NONE) { | |||||
GELOGW("Call rtProfilerStop failed, ret:%d", rt_ret); | |||||
} | |||||
for (size_t i = 0; i < prof_handle_vec_.size(); ++i) { | |||||
int result = ProfMgrStop(prof_handle_vec_[i]); | |||||
if (result != 0) { | |||||
GELOGW("ProfMgr stop return fail:%d, handle:%p", result, prof_handle_vec_[i]); | |||||
} | |||||
} | |||||
vector<void *>().swap(prof_handle_vec_); | |||||
is_load_ = false; | |||||
recv_profiling_config_ = ""; | |||||
GELOGI("Stop Profiling success."); | GELOGI("Stop Profiling success."); | ||||
#endif | #endif | ||||
} | } | ||||
@@ -392,12 +204,6 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::StopProf | |||||
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::ProfilingTaskDescInfo( | FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::ProfilingTaskDescInfo( | ||||
uint32_t model_id, const std::vector<TaskDescInfo> &task_desc_info, const int32_t &device_id) { | uint32_t model_id, const std::vector<TaskDescInfo> &task_desc_info, const int32_t &device_id) { | ||||
#ifdef DAVINCI_SUPPORT_PROFILING | #ifdef DAVINCI_SUPPORT_PROFILING | ||||
Msprof::Engine::Reporter *reporter = PluginImpl::GetPluginReporter(); | |||||
if (reporter == nullptr) { | |||||
GELOGI("Profiling report is nullptr!"); | |||||
return; | |||||
} | |||||
std::string data; | std::string data; | ||||
for (const auto &task : task_desc_info) { | for (const auto &task : task_desc_info) { | ||||
std::string model_name = task.model_name; | std::string model_name = task.model_name; | ||||
@@ -412,7 +218,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::Profilin | |||||
.append(std::to_string(stream_id)).append(" ") | .append(std::to_string(stream_id)).append(" ") | ||||
.append(std::to_string(model_id)).append("\n")); | .append(std::to_string(model_id)).append("\n")); | ||||
Msprof::Engine::ReporterData reporter_data{}; | |||||
ReporterData reporter_data{}; | |||||
reporter_data.deviceId = device_id; | reporter_data.deviceId = device_id; | ||||
reporter_data.data = (unsigned char *)data.c_str(); | reporter_data.data = (unsigned char *)data.c_str(); | ||||
reporter_data.dataLen = data.size(); | reporter_data.dataLen = data.size(); | ||||
@@ -422,9 +228,9 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::Profilin | |||||
return; | return; | ||||
} | } | ||||
ret = reporter->Report(&reporter_data); | |||||
if (ret != SUCCESS) { | |||||
GELOGE(ret, "Reporter data of task_desc_info fail!"); | |||||
int32_t cb_ret = CallMsprofReport(reporter_data); | |||||
if (cb_ret != 0) { | |||||
GELOGE(cb_ret, "Reporter data of task_desc_info failed, ret:%d", cb_ret); | |||||
return; | return; | ||||
} | } | ||||
} | } | ||||
@@ -436,9 +242,6 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::Profilin | |||||
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::ProfilingGraphDescInfo( | FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::ProfilingGraphDescInfo( | ||||
uint32_t model_id, const std::vector<ComputeGraphDescInfo> &compute_graph_desc_info, const int32_t &device_id) { | uint32_t model_id, const std::vector<ComputeGraphDescInfo> &compute_graph_desc_info, const int32_t &device_id) { | ||||
#ifdef DAVINCI_SUPPORT_PROFILING | #ifdef DAVINCI_SUPPORT_PROFILING | ||||
Msprof::Engine::Reporter *reporter = PluginImpl::GetPluginReporter(); | |||||
GE_IF_BOOL_EXEC(reporter == nullptr, GELOGI("Profiling report is nullptr!"); return;); | |||||
std::string data; | std::string data; | ||||
for (const auto &graph : compute_graph_desc_info) { | for (const auto &graph : compute_graph_desc_info) { | ||||
data.append("model_name:") | data.append("model_name:") | ||||
@@ -493,64 +296,52 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::Profilin | |||||
} | } | ||||
data.append(" model_id:").append(std::to_string(model_id)); | data.append(" model_id:").append(std::to_string(model_id)); | ||||
data.append("\n"); | data.append("\n"); | ||||
Msprof::Engine::ReporterData reporter_data{}; | |||||
Report(device_id, data, *reporter, reporter_data); | |||||
GraphDescReport(device_id, data); | |||||
data.clear(); | data.clear(); | ||||
} | } | ||||
#endif | #endif | ||||
} | } | ||||
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::Report( | |||||
const int32_t &device_id, const string &data, Msprof::Engine::Reporter &reporter, | |||||
Msprof::Engine::ReporterData &reporter_data) { | |||||
void ProfilingManager::GraphDescReport(const int32_t &device_id, const string &data) { | |||||
#ifdef DAVINCI_SUPPORT_PROFILING | #ifdef DAVINCI_SUPPORT_PROFILING | ||||
ReporterData reporter_data{}; | |||||
int ret = -1; | |||||
int32_t cb_ret = -1; | |||||
size_t index = data.size() / kReportMaxLen; | size_t index = data.size() / kReportMaxLen; | ||||
if (index >= 1) { | if (index >= 1) { | ||||
reporter_data.deviceId = device_id; | reporter_data.deviceId = device_id; | ||||
int ret = memcpy_s(reporter_data.tag, MSPROF_ENGINE_MAX_TAG_LEN + 1, "graph_desc_info", sizeof("graph_desc_info")); | |||||
ret = memcpy_s(reporter_data.tag, MSPROF_ENGINE_MAX_TAG_LEN + 1, "graph_desc_info", sizeof("graph_desc_info")); | |||||
GE_IF_BOOL_EXEC(ret != EOK, GELOGE(ret, "Report data tag of graph_desc_info memcpy error!"); return;); | GE_IF_BOOL_EXEC(ret != EOK, GELOGE(ret, "Report data tag of graph_desc_info memcpy error!"); return;); | ||||
for (size_t i = 0; i < index; ++i) { | for (size_t i = 0; i < index; ++i) { | ||||
reporter_data.data = (unsigned char *)data.c_str() + kReportMaxLen * i; | reporter_data.data = (unsigned char *)data.c_str() + kReportMaxLen * i; | ||||
reporter_data.dataLen = kReportMaxLen; | reporter_data.dataLen = kReportMaxLen; | ||||
ret = reporter.Report(&reporter_data); | |||||
GE_IF_BOOL_EXEC(ret != SUCCESS, GELOGE(ret, "Reporter data of graph_desc_info fail!"); return;); | |||||
cb_ret = CallMsprofReport(reporter_data); | |||||
GE_IF_BOOL_EXEC(cb_ret != 0, GELOGE(cb_ret, "Reporter data of graph_desc_info failed, ret:%d", cb_ret); return;); | |||||
} | } | ||||
reporter_data.dataLen = data.size() - kReportMaxLen * index; | reporter_data.dataLen = data.size() - kReportMaxLen * index; | ||||
if (reporter_data.dataLen != 0) { | if (reporter_data.dataLen != 0) { | ||||
reporter_data.data = (unsigned char *)data.c_str() + kReportMaxLen * index; | reporter_data.data = (unsigned char *)data.c_str() + kReportMaxLen * index; | ||||
ret = reporter.Report(&reporter_data); | |||||
GE_IF_BOOL_EXEC(ret != SUCCESS, GELOGE(ret, "Reporter data of graph_desc_info fail!"); return;); | |||||
cb_ret = CallMsprofReport(reporter_data); | |||||
GE_IF_BOOL_EXEC(cb_ret != 0, GELOGE(cb_ret, "Reporter data of graph_desc_info failed, ret:%d", cb_ret); return;); | |||||
} | } | ||||
} else { | } else { | ||||
reporter_data.deviceId = device_id; | reporter_data.deviceId = device_id; | ||||
reporter_data.data = (unsigned char *)data.c_str(); | reporter_data.data = (unsigned char *)data.c_str(); | ||||
reporter_data.dataLen = data.size(); | reporter_data.dataLen = data.size(); | ||||
int ret = memcpy_s(reporter_data.tag, MSPROF_ENGINE_MAX_TAG_LEN + 1, "graph_desc_info", sizeof("graph_desc_info")); | |||||
ret = memcpy_s(reporter_data.tag, MSPROF_ENGINE_MAX_TAG_LEN + 1, "graph_desc_info", sizeof("graph_desc_info")); | |||||
GE_IF_BOOL_EXEC(ret != EOK, GELOGE(ret, "Report data tag of graph_desc_info memcpy error!"); return;); | GE_IF_BOOL_EXEC(ret != EOK, GELOGE(ret, "Report data tag of graph_desc_info memcpy error!"); return;); | ||||
ret = reporter.Report(&reporter_data); | |||||
GE_IF_BOOL_EXEC(ret != SUCCESS, GELOGE(ret, "Reporter data of graph_desc_info fail!"); return;); | |||||
} | |||||
#endif | |||||
} | |||||
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::PluginUnInit(const std::string &module) const { | |||||
#ifdef DAVINCI_SUPPORT_PROFILING | |||||
int ret = Msprof::Engine::UnInit(module); | |||||
if (ret != SUCCESS) { | |||||
GELOGE(ret, "profiling plugin uninit failed, ret:%d", ret); | |||||
cb_ret = CallMsprofReport(reporter_data); | |||||
GE_IF_BOOL_EXEC(cb_ret != 0, GELOGE(cb_ret, "Reporter data of graph_desc_info failed, ret:%d", cb_ret); return;); | |||||
} | } | ||||
#endif | #endif | ||||
} | } | ||||
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::ReportProfilingData( | FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::ReportProfilingData( | ||||
uint32_t model_id, const std::vector<TaskDescInfo> &task_desc_info, | uint32_t model_id, const std::vector<TaskDescInfo> &task_desc_info, | ||||
const std::vector<ComputeGraphDescInfo> &compute_graph_desc_info, | |||||
bool check_device) { | |||||
const std::vector<ComputeGraphDescInfo> &compute_graph_desc_info) { | |||||
#ifdef DAVINCI_SUPPORT_PROFILING | #ifdef DAVINCI_SUPPORT_PROFILING | ||||
int32_t logic_device_id = 0; | int32_t logic_device_id = 0; | ||||
rtError_t rt_ret = rtGetDevice(&logic_device_id); | rtError_t rt_ret = rtGetDevice(&logic_device_id); | ||||
@@ -559,13 +350,6 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::ReportPr | |||||
return; | return; | ||||
} | } | ||||
GELOGD("current logic_device_id:%d", logic_device_id); | GELOGD("current logic_device_id:%d", logic_device_id); | ||||
if (check_device) { | |||||
auto ret = std::find(device_id_.begin(), device_id_.end(), logic_device_id); | |||||
if (ret == device_id_.end()) { | |||||
GELOGE(FAILED, "get valid phy_device_id failed, profiling report failed."); | |||||
return; | |||||
} | |||||
} | |||||
GELOGD("start ProfilingTaskDescInfo."); | GELOGD("start ProfilingTaskDescInfo."); | ||||
ProfilingTaskDescInfo(model_id, task_desc_info, logic_device_id); | ProfilingTaskDescInfo(model_id, task_desc_info, logic_device_id); | ||||
GELOGD("start ProfilingGraphDescInfo."); | GELOGD("start ProfilingGraphDescInfo."); | ||||
@@ -574,11 +358,6 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::ReportPr | |||||
#endif | #endif | ||||
} | } | ||||
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::SetProfilingConfig( | |||||
const std::string &profiling_cfg) { | |||||
recv_profiling_config_ = profiling_cfg; | |||||
} | |||||
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY uint64_t ProfilingManager::GetProfilingModule() { | FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY uint64_t ProfilingManager::GetProfilingModule() { | ||||
uint64_t module = PROF_MODEL_EXECUTE_MASK | | uint64_t module = PROF_MODEL_EXECUTE_MASK | | ||||
PROF_RUNTIME_API_MASK | | PROF_RUNTIME_API_MASK | | ||||
@@ -594,9 +373,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY uint64_t ProfilingManager::GetP | |||||
return module; | return module; | ||||
} | } | ||||
void ProfilingManager::UpdateSubscribeDeviceModuleMap(std::string prof_type, | |||||
uint32_t device_id, | |||||
uint64_t module) { | |||||
void ProfilingManager::UpdateSubscribeDeviceModuleMap(std::string prof_type, uint32_t device_id, uint64_t module) { | |||||
#ifdef DAVINCI_SUPPORT_PROFILING | #ifdef DAVINCI_SUPPORT_PROFILING | ||||
if (prof_type == kProfModelSubscribe) { | if (prof_type == kProfModelSubscribe) { | ||||
if (subs_dev_module_.find(device_id) != subs_dev_module_.end()) { | if (subs_dev_module_.find(device_id) != subs_dev_module_.end()) { | ||||
@@ -608,9 +385,13 @@ void ProfilingManager::UpdateSubscribeDeviceModuleMap(std::string prof_type, | |||||
subs_dev_module_[device_id] = dev_info; | subs_dev_module_[device_id] = dev_info; | ||||
} | } | ||||
} else if (prof_type == kProfModelUnsubscribe) { | } else if (prof_type == kProfModelUnsubscribe) { | ||||
if (subs_dev_module_.find(device_id) != subs_dev_module_.end()) { | |||||
if (subs_dev_module_[device_id].subscribe_count > 0) { | |||||
subs_dev_module_[device_id].subscribe_count--; | |||||
auto iter = subs_dev_module_.find(device_id); | |||||
if (iter != subs_dev_module_.end()) { | |||||
if (iter->second.subscribe_count > 0) { | |||||
iter->second.subscribe_count--; | |||||
} | |||||
if (iter->second.subscribe_count == 0) { | |||||
subs_dev_module_.erase(iter); | |||||
} | } | ||||
} | } | ||||
} else { | } else { | ||||
@@ -626,10 +407,11 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfMo | |||||
uint64_t model_load_mask = module & PROF_MODEL_LOAD_MASK; | uint64_t model_load_mask = module & PROF_MODEL_LOAD_MASK; | ||||
if ((subscribe_count_ == 0) && (model_load_mask == PROF_MODEL_LOAD_MASK)) { | if ((subscribe_count_ == 0) && (model_load_mask == PROF_MODEL_LOAD_MASK)) { | ||||
// register framework to profiling | // register framework to profiling | ||||
int32_t result = Msprof::Engine::Init(GE_PROFILING_MODULE, &engine_); | |||||
if (result != SUCCESS) { | |||||
GELOGE(FAILED, "Register profiling engine failed."); | |||||
return FAILED; | |||||
// register Framework to profiling | |||||
int32_t cb_ret = PluginInit(); | |||||
if (cb_ret != 0) { | |||||
GELOGE(cb_ret, "profiling plugin init failed, ret:%d", cb_ret); | |||||
return cb_ret; | |||||
} | } | ||||
GELOGI("Prof subscribe: model load profiling on."); | GELOGI("Prof subscribe: model load profiling on."); | ||||
} | } | ||||
@@ -647,7 +429,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfMo | |||||
UpdateSubscribeDeviceModuleMap(kProfModelSubscribe, device[0], module); | UpdateSubscribeDeviceModuleMap(kProfModelSubscribe, device[0], module); | ||||
// Report profiling data | // Report profiling data | ||||
Status p_ret = davinci_model->ReportProfilingData(false); | |||||
Status p_ret = davinci_model->ReportProfilingData(); | |||||
if (p_ret != SUCCESS) { | if (p_ret != SUCCESS) { | ||||
GELOGE(p_ret, "Report profiling data failed."); | GELOGE(p_ret, "Report profiling data failed."); | ||||
return p_ret; | return p_ret; | ||||
@@ -672,6 +454,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfMo | |||||
auto iter = subs_dev_module_.find(device[0]); | auto iter = subs_dev_module_.find(device[0]); | ||||
if (iter != subs_dev_module_.end()) { | if (iter != subs_dev_module_.end()) { | ||||
if (subs_dev_module_[device[0]].subscribe_count == 1) { | if (subs_dev_module_[device[0]].subscribe_count == 1) { | ||||
// The same device_id, only stop at last time | |||||
rtError_t rt_ret = rtProfilerStop(subs_dev_module_[device[0]].module, dev_num, device); | rtError_t rt_ret = rtProfilerStop(subs_dev_module_[device[0]].module, dev_num, device); | ||||
if (rt_ret != RT_ERROR_NONE) { | if (rt_ret != RT_ERROR_NONE) { | ||||
GELOGE(FAILED, "Runtime profiler stop failed."); | GELOGE(FAILED, "Runtime profiler stop failed."); | ||||
@@ -679,15 +462,15 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfMo | |||||
} | } | ||||
} | } | ||||
UpdateSubscribeDeviceModuleMap(kProfModelUnsubscribe, device[0], subs_dev_module_[device[0]].module); | UpdateSubscribeDeviceModuleMap(kProfModelUnsubscribe, device[0], subs_dev_module_[device[0]].module); | ||||
} else { | |||||
GELOGE(FAILED, "The device_id:%u has not been subscribed, do not need to cancel.", device[0]); | |||||
return FAILED; | |||||
} | } | ||||
subscribe_count_--; | subscribe_count_--; | ||||
if (subscribe_count_ == 0) { | if (subscribe_count_ == 0) { | ||||
int32_t ret = Msprof::Engine::UnInit(GE_PROFILING_MODULE); | |||||
if (ret != SUCCESS) { | |||||
GELOGE(ret, "Profiling plugin uninit failed, ret:%d", ret); | |||||
return ret; | |||||
} | |||||
// profiling plugin uninit at last subscription | |||||
PluginUnInit(); | |||||
} | } | ||||
#endif | #endif | ||||
return SUCCESS; | return SUCCESS; | ||||
@@ -700,11 +483,12 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfIn | |||||
if (model_load_mask == PROF_MODEL_LOAD_MASK) { | if (model_load_mask == PROF_MODEL_LOAD_MASK) { | ||||
// register Framework to profiling | // register Framework to profiling | ||||
int32_t result = Msprof::Engine::Init(GE_PROFILING_MODULE, &engine_); | |||||
if (result != SUCCESS) { | |||||
GELOGE(FAILED, "Register profiling engine failed."); | |||||
return FAILED; | |||||
int32_t cb_ret = PluginInit(); | |||||
if (cb_ret != 0) { | |||||
GELOGE(cb_ret, "profiling plugin init failed, ret:%d", cb_ret); | |||||
return cb_ret; | |||||
} | } | ||||
int32_t device_num = -1; | int32_t device_num = -1; | ||||
rtError_t rt_ret = rtProfilerStart(model_load_mask, device_num, nullptr); | rtError_t rt_ret = rtProfilerStart(model_load_mask, device_num, nullptr); | ||||
if (rt_ret != RT_ERROR_NONE) { | if (rt_ret != RT_ERROR_NONE) { | ||||
@@ -719,7 +503,6 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfIn | |||||
if (training_trace_mask == PROF_TRAINING_TRACE_MASK) { | if (training_trace_mask == PROF_TRAINING_TRACE_MASK) { | ||||
is_training_trace_ = true; | is_training_trace_ = true; | ||||
} | } | ||||
is_acl_api_mode_ = true; | |||||
GELOGI("Prof init success."); | GELOGI("Prof init success."); | ||||
#endif | #endif | ||||
return SUCCESS; | return SUCCESS; | ||||
@@ -730,19 +513,17 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfFi | |||||
std::lock_guard<std::mutex> lock(mutex_); | std::lock_guard<std::mutex> lock(mutex_); | ||||
is_load_profiling_ = false; | is_load_profiling_ = false; | ||||
is_training_trace_ = false; | is_training_trace_ = false; | ||||
is_acl_api_mode_ = false; | |||||
is_execute_profiling_ = false; | |||||
// profiling plugin uninit | |||||
PluginUnInit(); | |||||
int32_t ret = Msprof::Engine::UnInit(GE_PROFILING_MODULE); | |||||
if (ret != SUCCESS) { | |||||
GELOGE(ret, "Profiling plugin uninit failed, ret:%d", ret); | |||||
} | |||||
int32_t dev_num = -1; | int32_t dev_num = -1; | ||||
rtError_t rt_ret = rtProfilerStop(PROF_MODEL_LOAD_MASK, dev_num, nullptr); | rtError_t rt_ret = rtProfilerStop(PROF_MODEL_LOAD_MASK, dev_num, nullptr); | ||||
if (rt_ret != RT_ERROR_NONE) { | if (rt_ret != RT_ERROR_NONE) { | ||||
GELOGE(FAILED, "Runtime profiler stop failed."); | GELOGE(FAILED, "Runtime profiler stop failed."); | ||||
return FAILED; | return FAILED; | ||||
} | } | ||||
for (auto device_id_module : device_id_module_map_) { | for (auto device_id_module : device_id_module_map_) { | ||||
if (device_id_module.second != 0) { | if (device_id_module.second != 0) { | ||||
uint32_t device_id = static_cast<uint32_t>(device_id_module.first); | uint32_t device_id = static_cast<uint32_t>(device_id_module.first); | ||||
@@ -792,6 +573,7 @@ Status ProfilingManager::ProfParseDeviceId(const std::map<std::string, std::stri | |||||
return FAILED; | return FAILED; | ||||
} catch (std::out_of_range &) { | } catch (std::out_of_range &) { | ||||
GELOGE(FAILED, "Device id: %s is out of range.", decvice_id[i].c_str()); | GELOGE(FAILED, "Device id: %s is out of range.", decvice_id[i].c_str()); | ||||
return FAILED; | |||||
} catch (...) { | } catch (...) { | ||||
GELOGE(FAILED, "Device id: %s cannot change to int.", decvice_id[i].c_str()); | GELOGE(FAILED, "Device id: %s cannot change to int.", decvice_id[i].c_str()); | ||||
return FAILED; | return FAILED; | ||||
@@ -818,6 +600,7 @@ Status ProfilingManager::ProfParseParam(const std::map<std::string, std::string> | |||||
return FAILED; | return FAILED; | ||||
} catch (std::out_of_range &) { | } catch (std::out_of_range &) { | ||||
GELOGE(FAILED, "Device num: %s is out of range.", iter->second.c_str()); | GELOGE(FAILED, "Device num: %s is out of range.", iter->second.c_str()); | ||||
return FAILED; | |||||
} catch (...) { | } catch (...) { | ||||
GELOGE(FAILED, "Device num: %s cannot change to int.", iter->second.c_str()); | GELOGE(FAILED, "Device num: %s cannot change to int.", iter->second.c_str()); | ||||
return FAILED; | return FAILED; | ||||
@@ -859,7 +642,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfSt | |||||
for (int32_t i = 0; i < device_num; i++) { | for (int32_t i = 0; i < device_num; i++) { | ||||
device_id_ptr[i] = static_cast<uint32_t>(device_list[i]); | device_id_ptr[i] = static_cast<uint32_t>(device_list[i]); | ||||
} | } | ||||
GELOGD("Runtime config param: 0x%llx, device num: %d.", module, device_num); | |||||
GELOGI("Runtime config param: 0x%llx, device num: %d.", module, device_num); | |||||
rtError_t rt_ret = rtProfilerStart(module, device_num, device_id_ptr.get()); | rtError_t rt_ret = rtProfilerStart(module, device_num, device_id_ptr.get()); | ||||
if (rt_ret != RT_ERROR_NONE) { | if (rt_ret != RT_ERROR_NONE) { | ||||
@@ -878,7 +661,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfSt | |||||
GELOGW("Prof start: load model module is invalid."); | GELOGW("Prof start: load model module is invalid."); | ||||
} | } | ||||
UpdateDeviceIdModuleMap(kProfStart, module, device_list); | UpdateDeviceIdModuleMap(kProfStart, module, device_list); | ||||
GELOGD("Prof start profiling success."); | |||||
GELOGI("Prof start profiling success."); | |||||
#endif | #endif | ||||
return SUCCESS; | return SUCCESS; | ||||
} | } | ||||
@@ -901,7 +684,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfSt | |||||
for (int32_t i = 0; i < device_num; i++) { | for (int32_t i = 0; i < device_num; i++) { | ||||
device_id_ptr[i] = static_cast<uint32_t>(device_list[i]); | device_id_ptr[i] = static_cast<uint32_t>(device_list[i]); | ||||
} | } | ||||
GELOGD("Prof stop: runtime config param: 0x%llx, device num: %d", module, device_num); | |||||
GELOGI("Prof stop: runtime config param: 0x%llx, device num: %d", module, device_num); | |||||
rtError_t rt_ret = rtProfilerStop(module, device_num, device_id_ptr.get()); | rtError_t rt_ret = rtProfilerStop(module, device_num, device_id_ptr.get()); | ||||
if (rt_ret != RT_ERROR_NONE) { | if (rt_ret != RT_ERROR_NONE) { | ||||
GELOGE(FAILED, "Prof stop: runtime profiler config proc failed."); | GELOGE(FAILED, "Prof stop: runtime profiler config proc failed."); | ||||
@@ -921,7 +704,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfSt | |||||
GELOGW("Prof stop: load model module is invalid."); | GELOGW("Prof stop: load model module is invalid."); | ||||
} | } | ||||
UpdateDeviceIdModuleMap(kProfStop, module, device_list); | UpdateDeviceIdModuleMap(kProfStop, module, device_list); | ||||
GELOGD("Prof stop profiling success."); | |||||
GELOGI("Prof stop profiling success."); | |||||
#endif | #endif | ||||
return SUCCESS; | return SUCCESS; | ||||
} | } | ||||
@@ -963,47 +746,90 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY bool ProfilingManager::Profilin | |||||
if (rt_ret != RT_ERROR_NONE) { | if (rt_ret != RT_ERROR_NONE) { | ||||
GELOGE(rt_ret, "Runtime get logic_device_id failed, current logic_device_id:%d", logic_device_id); | GELOGE(rt_ret, "Runtime get logic_device_id failed, current logic_device_id:%d", logic_device_id); | ||||
} | } | ||||
GELOGD("Current logic_device_id:%d", logic_device_id); | |||||
GELOGI("Current logic_device_id:%d", logic_device_id); | |||||
bool execute_model_prof_on = false; | bool execute_model_prof_on = false; | ||||
auto iter = std::find(device_id_.begin(), device_id_.end(), logic_device_id); | auto iter = std::find(device_id_.begin(), device_id_.end(), logic_device_id); | ||||
if (iter != device_id_.end()) { | if (iter != device_id_.end()) { | ||||
execute_model_prof_on = true; | execute_model_prof_on = true; | ||||
} | } | ||||
GELOGD("Flag is_execute_profiling: %d, execute_model_prof_on: %d", is_execute_profiling_, execute_model_prof_on); | |||||
return is_execute_profiling_ || execute_model_prof_on; | |||||
GELOGI("Flag is_execute_profiling: %d, execute_model_prof_on: %d", is_execute_profiling_, execute_model_prof_on); | |||||
return execute_model_prof_on; | |||||
} | } | ||||
/** | |||||
* @brief Profiling PluginImpl | |||||
*/ | |||||
// PluginImpl static variable init | |||||
Msprof::Engine::Reporter *PluginImpl::reporter_ = nullptr; | |||||
PluginImpl::PluginImpl(const std::string &module) : module_(module) { GELOGI("Create PluginImpl\n"); } | |||||
int PluginImpl::Init(const Msprof::Engine::Reporter *reporter) { | |||||
GELOGI("PluginImpl init"); | |||||
reporter_ = const_cast<Msprof::Engine::Reporter *>(reporter); | |||||
return 0; | |||||
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::PluginInit() const { | |||||
if (prof_cb_.msprofReporterCallback == nullptr) { | |||||
GELOGE(ge::PARAM_INVALID, "MsprofReporterCallback callback is nullptr."); | |||||
return ge::PARAM_INVALID; | |||||
} | |||||
return prof_cb_.msprofReporterCallback( | |||||
static_cast<uint32_t>(MsprofReporterModuleId::MSPROF_MODULE_FRAMEWORK), | |||||
static_cast<uint32_t>(MsprofReporterCallbackType::MSPROF_REPORTER_INIT), | |||||
nullptr, 0); | |||||
} | } | ||||
int PluginImpl::UnInit() { | |||||
GELOGI("PluginImpl Uninit"); | |||||
reporter_ = nullptr; | |||||
return 0; | |||||
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::PluginUnInit() const { | |||||
#ifdef DAVINCI_SUPPORT_PROFILING | |||||
if (prof_cb_.msprofReporterCallback == nullptr) { | |||||
GELOGE(ge::PARAM_INVALID, "MsprofReporterCallback callback is nullptr."); | |||||
return; | |||||
} | |||||
int32_t cb_ret = prof_cb_.msprofReporterCallback( | |||||
static_cast<uint32_t>(MsprofReporterModuleId::MSPROF_MODULE_FRAMEWORK), | |||||
static_cast<uint32_t>(MsprofReporterCallbackType::MSPROF_REPORTER_UNINIT), | |||||
nullptr, 0); | |||||
if (cb_ret != 0) { | |||||
GELOGW("profiling plugin uninit failed, ret:%d", cb_ret); | |||||
} | |||||
#endif | |||||
} | } | ||||
Msprof::Engine::PluginIntf *ProfilingEngineImpl::CreatePlugin() { | |||||
GELOGI(" Create Plugin"); | |||||
return new (std::nothrow) PluginImpl(GE_PROFILING_MODULE); | |||||
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::CallMsprofReport( | |||||
ReporterData &reporter_data) const { | |||||
if (prof_cb_.msprofReporterCallback == nullptr) { | |||||
GELOGE(ge::PARAM_INVALID, "MsprofReporterCallback callback is nullptr."); | |||||
return ge::PARAM_INVALID; | |||||
} | |||||
return prof_cb_.msprofReporterCallback( | |||||
static_cast<uint32_t>(MsprofReporterModuleId::MSPROF_MODULE_FRAMEWORK), | |||||
static_cast<uint32_t>(MsprofReporterCallbackType::MSPROF_REPORTER_REPORT), | |||||
static_cast<void *>(&reporter_data), sizeof(ReporterData)); | |||||
} | } | ||||
int ProfilingEngineImpl::ReleasePlugin(Msprof::Engine::PluginIntf *plugin) { | |||||
if (plugin != nullptr) { | |||||
delete plugin; | |||||
plugin = nullptr; | |||||
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::GetFpBpPoint( | |||||
std::string &fp_point, std::string &bp_point) { | |||||
// Env or options mode, fp_point_/bp_point_ have initiliazed on profiling init | |||||
if (!fp_point_.empty() && !bp_point_.empty()) { | |||||
fp_point = fp_point_; | |||||
bp_point = bp_point_; | |||||
GELOGI("Bp Fp have been initialized in env or options. bp_point: %s, fp_point: %s", bp_point.c_str(), fp_point.c_str()); | |||||
return; | |||||
} | |||||
// ProfApi mode and training trace is set | |||||
try { | |||||
char env_profiling_options[MSPROF_OPTIONS_DEF_LEN_MAX] = { 0x00 }; | |||||
INT32 ret = mmGetEnv("PROFILING_OPTIONS", env_profiling_options, MSPROF_OPTIONS_DEF_LEN_MAX); | |||||
if (ret != EN_OK) { | |||||
GELOGI("PROFILING_OPTIONS env is not exist."); | |||||
return; | |||||
} | |||||
GELOGI("Parse env PROFILING_OPTIONS:%s.", env_profiling_options); | |||||
Json prof_options = Json::parse(env_profiling_options); | |||||
fp_point_ = prof_options[kFpPoint]; | |||||
bp_point_ = prof_options[kBpPoint]; | |||||
fp_point = fp_point_; | |||||
bp_point = bp_point_; | |||||
if (!fp_point_.empty() && !bp_point_.empty()) { | |||||
GELOGI("Training trace bp fp is set, bp_point:%s, fp_point:%s.", bp_point_.c_str(), fp_point_.c_str()); | |||||
} | |||||
} catch (...) { | |||||
GELOGE(FAILED, "Json prof options is invalid."); | |||||
return; | |||||
} | } | ||||
return 0; | |||||
return; | |||||
} | } | ||||
} // namespace ge | } // namespace ge |
@@ -26,9 +26,7 @@ | |||||
#include "framework/common/ge_inner_error_codes.h" | #include "framework/common/ge_inner_error_codes.h" | ||||
#include "framework/common/ge_types.h" | #include "framework/common/ge_types.h" | ||||
#include "external/register/register_types.h" | #include "external/register/register_types.h" | ||||
#include "toolchain/prof_engine.h" | |||||
#include "toolchain/prof_mgr_core.h" | |||||
#include "toolchain/prof_acl_api.h" | |||||
#include "toolchain/prof_callback.h" | |||||
using std::map; | using std::map; | ||||
using std::string; | using std::string; | ||||
@@ -37,35 +35,33 @@ using Json = nlohmann::json; | |||||
namespace { | namespace { | ||||
const std::string GE_PROFILING_MODULE = "Framework"; | const std::string GE_PROFILING_MODULE = "Framework"; | ||||
// DataTypeConfig MASK | |||||
#define PROF_ACL_API_MASK 0x0001 | |||||
#define PROF_TASK_TIME_MASK 0x0002 | |||||
#define PROF_AICORE_METRICS_MASK 0x0004 | |||||
#define PROF_AICPU_TRACE_MASK 0x0008 | |||||
#define PROF_MODEL_EXECUTE_MASK 0x0010 | |||||
#define PROF_RUNTIME_API_MASK 0x0020 | |||||
#define PROF_RUNTIME_TRACE_MASK 0x0040 | |||||
#define PROF_SCHEDULE_TIMELINE_MASK 0x0080 | |||||
#define PROF_SCHEDULE_TRACE_MASK 0x0100 | |||||
#define PROF_AIVECTORCORE_METRICS_MASK 0x0200 | |||||
#define PROF_SUBTASK_TIME_MASK 0x0400 | |||||
#define PROF_TRAINING_TRACE_MASK 0x0800 | |||||
#define PROF_HCCL_TRACE_MASK 0x1000 | |||||
#define PROF_DATA_PROCESS_MASK 0x2000 | |||||
#define PROF_MODEL_LOAD_MASK 0x8000000000000000 | |||||
} // namespace | } // namespace | ||||
namespace ge { | namespace ge { | ||||
struct DeviceSubsInfo { | struct DeviceSubsInfo { | ||||
uint64_t module; | uint64_t module; | ||||
uint32_t subscribe_count; | uint32_t subscribe_count; | ||||
}; | }; | ||||
// register Plugin | |||||
class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY PluginImpl : public Msprof::Engine::PluginIntf { | |||||
public: | |||||
explicit PluginImpl(const std::string &module); | |||||
~PluginImpl() {} | |||||
int Init(const Msprof::Engine::Reporter *reporter); | |||||
int UnInit(); | |||||
static Msprof::Engine::Reporter *GetPluginReporter() { return reporter_; } | |||||
private: | |||||
static Msprof::Engine::Reporter *reporter_; | |||||
std::string module_; | |||||
}; | |||||
// register Engine | |||||
class ProfilingEngineImpl : public Msprof::Engine::EngineIntf { | |||||
public: | |||||
ProfilingEngineImpl() {} | |||||
~ProfilingEngineImpl() {} | |||||
Msprof::Engine::PluginIntf *CreatePlugin(); | |||||
int ReleasePlugin(Msprof::Engine::PluginIntf *plugin); | |||||
struct MsprofCallback { | |||||
MsprofCtrlCallback msprofCtrlCallback; | |||||
MsprofReporterCallback msprofReporterCallback; | |||||
}; | }; | ||||
class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ProfilingManager { | class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ProfilingManager { | ||||
@@ -73,68 +69,54 @@ class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ProfilingManager { | |||||
ProfilingManager(); | ProfilingManager(); | ||||
virtual ~ProfilingManager(); | virtual ~ProfilingManager(); | ||||
static ProfilingManager &Instance(); | static ProfilingManager &Instance(); | ||||
ge::Status Init(const Options &options); | |||||
ge::Status InitFromOptions(const Options &options); | |||||
ge::Status InitFromAclCfg(const std::string &config); | |||||
ge::Status StartProfiling(int32_t iter, int32_t device_id); | |||||
void UpdateSubscribeDeviceModuleMap(std::string prof_type, uint32_t device_id, uint64_t module); | |||||
ge::Status ProfModelSubscribe(uint64_t module, void *model); | |||||
ge::Status ProfModelUnsubscribe(void *model); | |||||
ge::Status ProfInit(uint64_t module); | |||||
ge::Status ProfFinalize(); | |||||
ge::Status ProfStartProfiling(uint64_t module, const std::map<std::string, std::string> &config_para); | |||||
ge::Status ProfStopProfiling(uint64_t module, const std::map<std::string, std::string> &config_para); | |||||
Status Init(const Options &options); | |||||
Status ProfInit(uint64_t module); | |||||
Status ProfFinalize(); | |||||
Status ProfStartProfiling(uint64_t module, const std::map<std::string, std::string> &config_para); | |||||
Status ProfStopProfiling(uint64_t module, const std::map<std::string, std::string> &config_para); | |||||
Status ProfModelSubscribe(uint64_t module, void *model); | |||||
Status ProfModelUnsubscribe(void *model); | |||||
void StopProfiling(); | void StopProfiling(); | ||||
bool ProfilingOpTraceOn() const { return is_op_trace_; } | |||||
bool ProfilingLoadFlag() const { return is_load_; } | |||||
bool ProfilingTrainingTraceOn() const { return is_training_trace_; } | bool ProfilingTrainingTraceOn() const { return is_training_trace_; } | ||||
bool ProfilingModelLoadOn() const { return is_load_profiling_; } | bool ProfilingModelLoadOn() const { return is_load_profiling_; } | ||||
bool ProfilingModelExecuteOn() const; | bool ProfilingModelExecuteOn() const; | ||||
bool ProfilingOn() const { return is_load_profiling_ && is_execute_profiling_; } // only used by command pattern | |||||
bool IsAclApiMode() const { return is_acl_api_mode_; } | |||||
int32_t GetOpTraceIterNum() const { return op_trace_iter_num_; } | |||||
bool ProfilingOn() const { return is_load_profiling_ && is_execute_profiling_; } // is_execute_profiling_ only used by ge option and env | |||||
void ReportProfilingData(uint32_t model_id, const std::vector<TaskDescInfo> &task_desc_info, | void ReportProfilingData(uint32_t model_id, const std::vector<TaskDescInfo> &task_desc_info, | ||||
const std::vector<ComputeGraphDescInfo> &compute_graph_desc_info, | |||||
bool check_device); | |||||
void Report(const int32_t &device_id, const string &data, Msprof::Engine::Reporter &reporter, | |||||
Msprof::Engine::ReporterData &reporter_data); | |||||
const std::vector<ComputeGraphDescInfo> &compute_graph_desc_info); | |||||
void ProfilingTaskDescInfo(uint32_t model_id, const std::vector<TaskDescInfo> &task_desc_info, | void ProfilingTaskDescInfo(uint32_t model_id, const std::vector<TaskDescInfo> &task_desc_info, | ||||
const int32_t &device_id); | const int32_t &device_id); | ||||
void ProfilingGraphDescInfo(uint32_t model_id, const std::vector<ComputeGraphDescInfo> &compute_graph_desc_info, | void ProfilingGraphDescInfo(uint32_t model_id, const std::vector<ComputeGraphDescInfo> &compute_graph_desc_info, | ||||
const int32_t &device_id); | const int32_t &device_id); | ||||
void SetProfilingConfig(const string &profiling_cfg); | |||||
vector<int32_t> GetProfilingDeviceId() const { return device_id_; } | |||||
void PluginUnInit(const std::string &module) const; | |||||
Status PluginInit() const; | |||||
void PluginUnInit() const; | |||||
Status CallMsprofReport(ReporterData &reporter_data) const; | |||||
struct MsprofCallback &GetMsprofCallback() { return prof_cb_; } | |||||
void SetMsprofCtrlCallback(MsprofCtrlCallback func) { prof_cb_.msprofCtrlCallback = func; } | |||||
void SetMsprofReporterCallback(MsprofReporterCallback func) { prof_cb_.msprofReporterCallback = func; } | |||||
void GetFpBpPoint(std::string &fp_point, std::string &bp_point); | |||||
private: | private: | ||||
ge::Status ParseFeaturesFromAclCfg(const Json &feature); | |||||
ge::Status ProfParseParam(const std::map<std::string, std::string> &config_para, int32_t &device_num, | |||||
vector<int32_t> &device_list); | |||||
ge::Status ProfParseDeviceId(const std::map<std::string, std::string> &config_para, | |||||
Status InitFromOptions(const Options &options, MsprofGeOptions &prof_conf); | |||||
Status ParseOptions(const std::string &options); | |||||
Status ProfParseParam(const std::map<std::string, std::string> &config_para, int32_t &device_num, | |||||
vector<int32_t> &device_list); | |||||
Status ProfParseDeviceId(const std::map<std::string, std::string> &config_para, | |||||
vector<int32_t> &device_list); | vector<int32_t> &device_list); | ||||
uint64_t GetProfilingModule(); | uint64_t GetProfilingModule(); | ||||
void GraphDescReport(const int32_t &device_id, const string &data); | |||||
void UpdateDeviceIdModuleMap(string prof_type, uint64_t module, const vector<int32_t> &device_list); | void UpdateDeviceIdModuleMap(string prof_type, uint64_t module, const vector<int32_t> &device_list); | ||||
bool is_load_profiling_ = false; | |||||
bool is_execute_profiling_ = false; | |||||
bool is_op_trace_ = false; | |||||
bool is_load_ = false; | |||||
bool is_training_trace_ = false; | |||||
bool is_acl_api_mode_ = false; | |||||
int32_t op_trace_iter_num_ = 0; | |||||
string job_id_; | |||||
string prof_dir_; | |||||
void UpdateSubscribeDeviceModuleMap(std::string prof_type, uint32_t device_id, uint64_t module); | |||||
bool is_load_profiling_; | |||||
bool is_execute_profiling_; | |||||
bool is_training_trace_; | |||||
vector<int32_t> device_id_; | vector<int32_t> device_id_; | ||||
vector<string> op_trace_conf_; | |||||
vector<string> profiling_opts_; | |||||
vector<void *> prof_handle_vec_; | |||||
string recv_profiling_config_; | |||||
string send_profiling_config_; | |||||
string system_trace_conf_; | |||||
string task_trace_conf_; | |||||
const ProfilingEngineImpl engine_; | |||||
map<int32_t, uint64_t> device_id_module_map_; // key: device_id, value: profiling on module | map<int32_t, uint64_t> device_id_module_map_; // key: device_id, value: profiling on module | ||||
map<uint32_t, DeviceSubsInfo> subs_dev_module_; // key: device_id, value: profiling on module | map<uint32_t, DeviceSubsInfo> subs_dev_module_; // key: device_id, value: profiling on module | ||||
uint32_t subscribe_count_; | uint32_t subscribe_count_; | ||||
std::mutex mutex_; | std::mutex mutex_; | ||||
MsprofCallback prof_cb_; | |||||
std::string fp_point_; | |||||
std::string bp_point_; | |||||
}; | }; | ||||
} // namespace ge | } // namespace ge | ||||
#endif // GE_COMMON_PROFILING_PROFILING_MANAGER_H_ | #endif // GE_COMMON_PROFILING_PROFILING_MANAGER_H_ |
@@ -17,6 +17,7 @@ set(SRC_LIST | |||||
"../common/dump/dump_properties.cc" | "../common/dump/dump_properties.cc" | ||||
"../common/dump/dump_manager.cc" | "../common/dump/dump_manager.cc" | ||||
"../common/dump/dump_op.cc" | "../common/dump/dump_op.cc" | ||||
"../common/profiling/ge_profiling.cc" | |||||
"../graph/load/graph_loader.cc" | "../graph/load/graph_loader.cc" | ||||
"../graph/execute/graph_execute.cc" | "../graph/execute/graph_execute.cc" | ||||
"../omm/csa_interact.cc" | "../omm/csa_interact.cc" | ||||
@@ -244,7 +245,6 @@ target_link_libraries(ge_executor_shared PRIVATE | |||||
mmpa | mmpa | ||||
graph | graph | ||||
register | register | ||||
msprof | |||||
error_manager | error_manager | ||||
ascend_hal_stub | ascend_hal_stub | ||||
ascend_protobuf | ascend_protobuf | ||||
@@ -304,7 +304,7 @@ Status GeExecutor::Finalize() { | |||||
// Stop profiling | // Stop profiling | ||||
if (ProfilingManager::Instance().ProfilingOn()) { | if (ProfilingManager::Instance().ProfilingOn()) { | ||||
ProfilingManager::Instance().StopProfiling(); | ProfilingManager::Instance().StopProfiling(); | ||||
ProfilingManager::Instance().PluginUnInit(GE_PROFILING_MODULE); | |||||
ProfilingManager::Instance().PluginUnInit(); | |||||
} | } | ||||
GELOGI("Uninit GeExecutor over."); | GELOGI("Uninit GeExecutor over."); | ||||
@@ -8,6 +8,7 @@ local_ge_executor_src_files := \ | |||||
../common/dump/dump_op.cc \ | ../common/dump/dump_op.cc \ | ||||
../common/ge/plugin_manager.cc \ | ../common/ge/plugin_manager.cc \ | ||||
../common/ge/op_tiling_manager.cc \ | ../common/ge/op_tiling_manager.cc \ | ||||
../common/profiling/ge_profiling.cc \ | |||||
../graph/load/graph_loader.cc \ | ../graph/load/graph_loader.cc \ | ||||
../graph/execute/graph_execute.cc \ | ../graph/execute/graph_execute.cc \ | ||||
../omm/csa_interact.cc \ | ../omm/csa_interact.cc \ | ||||
@@ -177,7 +178,6 @@ local_ge_executor_shared_library := \ | |||||
libmmpa \ | libmmpa \ | ||||
libgraph \ | libgraph \ | ||||
libregister \ | libregister \ | ||||
libmsprof \ | |||||
liberror_manager \ | liberror_manager \ | ||||
local_ge_executor_ldflags := -lrt -ldl \ | local_ge_executor_ldflags := -lrt -ldl \ | ||||
@@ -234,7 +234,6 @@ LOCAL_SHARED_LIBRARIES := \ | |||||
libmmpa \ | libmmpa \ | ||||
libgraph \ | libgraph \ | ||||
libregister \ | libregister \ | ||||
libmsprof \ | |||||
liberror_manager \ | liberror_manager \ | ||||
stub/libascend_hal \ | stub/libascend_hal \ | ||||
@@ -272,7 +271,6 @@ LOCAL_SHARED_LIBRARIES := \ | |||||
libruntime \ | libruntime \ | ||||
libslog \ | libslog \ | ||||
libmmpa \ | libmmpa \ | ||||
libmsprof \ | |||||
LOCAL_LDFLAGS += $(local_ge_executor_ldflags) | LOCAL_LDFLAGS += $(local_ge_executor_ldflags) | ||||
@@ -304,7 +302,6 @@ LOCAL_SHARED_LIBRARIES := \ | |||||
libruntime \ | libruntime \ | ||||
libslog \ | libslog \ | ||||
libmmpa \ | libmmpa \ | ||||
libmsprof \ | |||||
ifeq ($(device_os),android) | ifeq ($(device_os),android) | ||||
LOCAL_LDFLAGS += -ldl | LOCAL_LDFLAGS += -ldl | ||||
@@ -29,6 +29,8 @@ LIBGE_LOCAL_SRC_FILES := \ | |||||
common/dump/dump_manager.cc \ | common/dump/dump_manager.cc \ | ||||
common/dump/dump_properties.cc \ | common/dump/dump_properties.cc \ | ||||
common/dump/dump_op.cc \ | common/dump/dump_op.cc \ | ||||
common/profiling/ge_profiling.cc \ | |||||
common/profiling/ge_runner_profiling.cc \ | |||||
engine_manager/dnnengine_manager.cc \ | engine_manager/dnnengine_manager.cc \ | ||||
ge_local_engine/engine/host_cpu_engine.cc \ | ge_local_engine/engine/host_cpu_engine.cc \ | ||||
generator/ge_generator.cc \ | generator/ge_generator.cc \ | ||||
@@ -307,7 +309,6 @@ LIBGE_LOCAL_SRC_FILES := \ | |||||
LIBCLIENT_LOCAL_SRC_FILES := \ | LIBCLIENT_LOCAL_SRC_FILES := \ | ||||
proto/ge_api.proto \ | proto/ge_api.proto \ | ||||
client/ge_api.cc \ | client/ge_api.cc \ | ||||
client/ge_prof.cc \ | |||||
RUNNER_LOCAL_C_INCLUDES := \ | RUNNER_LOCAL_C_INCLUDES := \ | ||||
$(LOCAL_PATH) ./ \ | $(LOCAL_PATH) ./ \ | ||||
@@ -372,7 +373,7 @@ LOCAL_SRC_FILES += $(LIBCLIENT_LOCAL_SRC_FILES) | |||||
LOCAL_STATIC_LIBRARIES := libge_memory \ | LOCAL_STATIC_LIBRARIES := libge_memory \ | ||||
libadump_server \ | libadump_server \ | ||||
libmsprofiler \ | |||||
libmsprofiler_fwk \ | |||||
libmmpa \ | libmmpa \ | ||||
LOCAL_SHARED_LIBRARIES := \ | LOCAL_SHARED_LIBRARIES := \ | ||||
@@ -382,7 +383,6 @@ LOCAL_SHARED_LIBRARIES := \ | |||||
libgraph \ | libgraph \ | ||||
libregister \ | libregister \ | ||||
libge_common \ | libge_common \ | ||||
libmsprof \ | |||||
liberror_manager \ | liberror_manager \ | ||||
LOCAL_LDFLAGS := -lrt -ldl | LOCAL_LDFLAGS := -lrt -ldl | ||||
@@ -409,7 +409,6 @@ endif | |||||
LOCAL_C_INCLUDES := $(RUNNER_LOCAL_C_INCLUDES) | LOCAL_C_INCLUDES := $(RUNNER_LOCAL_C_INCLUDES) | ||||
LOCAL_SRC_FILES := ../../out/ge/lib64/stub/ge_api.cc \ | LOCAL_SRC_FILES := ../../out/ge/lib64/stub/ge_api.cc \ | ||||
../../out/ge/lib64/stub/ge_prof.cc \ | |||||
../../out/ge/lib64/stub/ge_ir_build.cc \ | ../../out/ge/lib64/stub/ge_ir_build.cc \ | ||||
LOCAL_SHARED_LIBRARIES := | LOCAL_SHARED_LIBRARIES := | ||||
@@ -465,7 +464,6 @@ LOCAL_SHARED_LIBRARIES := \ | |||||
libc_sec \ | libc_sec \ | ||||
libslog \ | libslog \ | ||||
libmmpa \ | libmmpa \ | ||||
libmsprof \ | |||||
LOCAL_LDFLAGS := -lrt -ldl | LOCAL_LDFLAGS := -lrt -ldl | ||||
@@ -498,7 +496,6 @@ LOCAL_SHARED_LIBRARIES := \ | |||||
libc_sec \ | libc_sec \ | ||||
libslog \ | libslog \ | ||||
libmmpa \ | libmmpa \ | ||||
libmsprof \ | |||||
LOCAL_LDFLAGS := -lrt -ldl | LOCAL_LDFLAGS := -lrt -ldl | ||||
@@ -49,8 +49,6 @@ const char *const kIsLastNode = "is_last_node"; | |||||
const char *const kIsInputVar = "INPUT_IS_VAR"; | const char *const kIsInputVar = "INPUT_IS_VAR"; | ||||
const char *const kIsOutputVar = "OUTPUT_IS_VAR"; | const char *const kIsOutputVar = "OUTPUT_IS_VAR"; | ||||
const char *const kProfilingMode = "PROFILING_MODE"; | const char *const kProfilingMode = "PROFILING_MODE"; | ||||
const char *const kProfilingFpPoint = "FP_POINT"; | |||||
const char *const kProfilingBpPoint = "BP_POINT"; | |||||
const uint32_t kProfilingArStep = 2; | const uint32_t kProfilingArStep = 2; | ||||
const uint64_t kProfilingFpStartLogid = 1; | const uint64_t kProfilingFpStartLogid = 1; | ||||
const uint64_t kProfilingBpEndLogid = 2; | const uint64_t kProfilingBpEndLogid = 2; | ||||
@@ -810,35 +808,23 @@ Status TaskGenerator::GetFpBpIndex(const ComputeGraphPtr &graph, ProfilingPoint | |||||
vector<uint32_t> &all_reduce_nodes, std::string &fp_point_str, | vector<uint32_t> &all_reduce_nodes, std::string &fp_point_str, | ||||
std::string &bp_point_str) const { | std::string &bp_point_str) const { | ||||
if (ge::GetContext().GetOption(OPTION_EXEC_PROFILING_FPPONIT_OPTIONS, fp_point_str) == SUCCESS && | |||||
ge::GetContext().GetOption(OPTION_EXEC_PROFILING_BPPONIT_OPTIONS, bp_point_str) == SUCCESS && | |||||
!fp_point_str.empty() && !bp_point_str.empty()) { | |||||
return SUCCESS; | |||||
} | |||||
ProfilingManager::Instance().GetFpBpPoint(fp_point_str, bp_point_str); | |||||
Status ret = SUCCESS; | Status ret = SUCCESS; | ||||
const char *fp_point = std::getenv(kProfilingFpPoint); | |||||
if (fp_point == nullptr) { | |||||
if (fp_point_str.empty()) { | |||||
ret = AutoFindFpOpIndex(graph, profiling_point); | ret = AutoFindFpOpIndex(graph, profiling_point); | ||||
if (ret != SUCCESS) { | if (ret != SUCCESS) { | ||||
GELOGW("First forward profiling op_index not set and FindFpOpIndex failed."); | GELOGW("First forward profiling op_index not set and FindFpOpIndex failed."); | ||||
return FAILED; | return FAILED; | ||||
} | } | ||||
} else { | |||||
fp_point_str = string(fp_point); | |||||
GELOGI("Get fp_point_str from env %s", fp_point_str.c_str()); | |||||
} | } | ||||
const char *bp_point = std::getenv(kProfilingBpPoint); | |||||
if (bp_point == nullptr) { | |||||
if (bp_point_str.empty()) { | |||||
ret = AutoFindBpOpIndex(graph, profiling_point, all_reduce_nodes); | ret = AutoFindBpOpIndex(graph, profiling_point, all_reduce_nodes); | ||||
if (ret != SUCCESS) { | if (ret != SUCCESS) { | ||||
GELOGW("Last backward profiling op_index not set and FindBpOpIndex failed."); | GELOGW("Last backward profiling op_index not set and FindBpOpIndex failed."); | ||||
return FAILED; | return FAILED; | ||||
} | } | ||||
} else { | |||||
bp_point_str = string(bp_point); | |||||
GELOGI("Get bp_point_str from env %s", bp_point_str.c_str()); | |||||
} | } | ||||
return SUCCESS; | return SUCCESS; | ||||
@@ -712,7 +712,7 @@ Status DavinciModel::Init(void *dev_ptr, size_t mem_size, void *weight_ptr, size | |||||
// collect profiling for ge | // collect profiling for ge | ||||
auto &profiling_manager = ProfilingManager::Instance(); | auto &profiling_manager = ProfilingManager::Instance(); | ||||
if (profiling_manager.ProfilingModelLoadOn()) { | if (profiling_manager.ProfilingModelLoadOn()) { | ||||
Status p_ret = ReportProfilingData(!profiling_manager.IsAclApiMode()); | |||||
Status p_ret = ReportProfilingData(); | |||||
if (p_ret != SUCCESS) { | if (p_ret != SUCCESS) { | ||||
GELOGE(p_ret, "Report profiling data failed."); | GELOGE(p_ret, "Report profiling data failed."); | ||||
return p_ret; | return p_ret; | ||||
@@ -723,14 +723,14 @@ Status DavinciModel::Init(void *dev_ptr, size_t mem_size, void *weight_ptr, size | |||||
return ret; | return ret; | ||||
} | } | ||||
Status DavinciModel::ReportProfilingData(bool check_device) { | |||||
Status DavinciModel::ReportProfilingData() { | |||||
std::vector<ComputeGraphDescInfo> compute_graph_desc_info; | std::vector<ComputeGraphDescInfo> compute_graph_desc_info; | ||||
Status ret = GetComputeGraphInfo(compute_graph_desc_info); | Status ret = GetComputeGraphInfo(compute_graph_desc_info); | ||||
if (ret != SUCCESS) { | if (ret != SUCCESS) { | ||||
GELOGE(ret, "GetComputeGraphInfo failed."); | GELOGE(ret, "GetComputeGraphInfo failed."); | ||||
return ret; | return ret; | ||||
} | } | ||||
ProfilingManager::Instance().ReportProfilingData(model_id_, GetTaskDescInfo(), compute_graph_desc_info, check_device); | |||||
ProfilingManager::Instance().ReportProfilingData(model_id_, GetTaskDescInfo(), compute_graph_desc_info); | |||||
GE_CHK_STATUS(SinkModelProfile(), "Sink model profiler failed."); | GE_CHK_STATUS(SinkModelProfile(), "Sink model profiler failed."); | ||||
op_list_.clear(); | op_list_.clear(); | ||||
@@ -2250,10 +2250,8 @@ inline int64_t SumSize(const vector<int64_t> &size_list) { | |||||
Status DavinciModel::SinkModelProfile() { | Status DavinciModel::SinkModelProfile() { | ||||
// profiling plugin must be registered | // profiling plugin must be registered | ||||
Msprof::Engine::Reporter *reporter = PluginImpl::GetPluginReporter(); | |||||
GE_IF_BOOL_EXEC(reporter == nullptr, GELOGI("Profiling report is nullptr!"); return SUCCESS); | |||||
Msprof::Engine::ReporterData reporter_data{}; | |||||
auto &prof_mgr = ProfilingManager::Instance(); | |||||
ReporterData reporter_data{}; | |||||
// report model data tag name | // report model data tag name | ||||
std::string tag_name; | std::string tag_name; | ||||
tag_name.append("model_load_info_").append(std::to_string(this->Id())); | tag_name.append("model_load_info_").append(std::to_string(this->Id())); | ||||
@@ -2271,32 +2269,32 @@ Status DavinciModel::SinkModelProfile() { | |||||
reporter_data.deviceId = device_id_; | reporter_data.deviceId = device_id_; | ||||
reporter_data.data = (unsigned char *)&name_len; | reporter_data.data = (unsigned char *)&name_len; | ||||
reporter_data.dataLen = sizeof(int32_t); | reporter_data.dataLen = sizeof(int32_t); | ||||
GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", | |||||
this->Id()); | |||||
GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, | |||||
"Reporter data fail, model id:%u.", this->Id()); | |||||
reporter_data.data = (unsigned char *)name.c_str(); | reporter_data.data = (unsigned char *)name.c_str(); | ||||
reporter_data.dataLen = name.size(); | reporter_data.dataLen = name.size(); | ||||
GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", | |||||
this->Id()); | |||||
GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, | |||||
"Reporter data fail, model id:%u.", this->Id()); | |||||
uint32_t model_id = this->Id(); | uint32_t model_id = this->Id(); | ||||
reporter_data.data = (unsigned char *)&model_id; | reporter_data.data = (unsigned char *)&model_id; | ||||
reporter_data.dataLen = sizeof(uint32_t); | reporter_data.dataLen = sizeof(uint32_t); | ||||
GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", | |||||
this->Id()); | |||||
GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, | |||||
"Reporter data fail, model id:%u.", this->Id()); | |||||
// Load Start/End Time | // Load Start/End Time | ||||
int64_t start_time = this->GetLoadBeginTime(); | int64_t start_time = this->GetLoadBeginTime(); | ||||
reporter_data.data = (unsigned char *)&start_time; | reporter_data.data = (unsigned char *)&start_time; | ||||
reporter_data.dataLen = sizeof(int64_t); | reporter_data.dataLen = sizeof(int64_t); | ||||
GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", | |||||
this->Id()); | |||||
GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, | |||||
"Reporter data fail, model id:%u.", this->Id()); | |||||
int64_t end_time = this->GetLoadEndTime(); | int64_t end_time = this->GetLoadEndTime(); | ||||
reporter_data.data = (unsigned char *)&end_time; | reporter_data.data = (unsigned char *)&end_time; | ||||
reporter_data.dataLen = sizeof(int64_t); | reporter_data.dataLen = sizeof(int64_t); | ||||
GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", | |||||
this->Id()); | |||||
GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, | |||||
"Reporter data fail, model id:%u.", this->Id()); | |||||
int32_t task_num = task_list_.size(); | int32_t task_num = task_list_.size(); | ||||
std::multimap<uint32_t, uint32_t> op_id_map; | std::multimap<uint32_t, uint32_t> op_id_map; | ||||
@@ -2310,6 +2308,7 @@ Status DavinciModel::SinkModelProfile() { | |||||
uint32_t op_num = fusion_op_info->original_op_names.size(); | uint32_t op_num = fusion_op_info->original_op_names.size(); | ||||
uint32_t task_id = task->GetTaskID(); | uint32_t task_id = task->GetTaskID(); | ||||
if (op_num > 0) { | if (op_num > 0) { | ||||
GELOGI("task.id = %u, opNum = %u", task_id, op_num); | |||||
op_id_map.insert(std::make_pair(fusion_op_info->op_index, task_id)); | op_id_map.insert(std::make_pair(fusion_op_info->op_index, task_id)); | ||||
} | } | ||||
} | } | ||||
@@ -2352,39 +2351,39 @@ Status DavinciModel::SinkModelProfile() { | |||||
int32_t fusion_op_name_len = fusion_op_name.size() == 0 ? 1 : fusion_op_name.size(); | int32_t fusion_op_name_len = fusion_op_name.size() == 0 ? 1 : fusion_op_name.size(); | ||||
reporter_data.data = (unsigned char *)&fusion_op_name_len; | reporter_data.data = (unsigned char *)&fusion_op_name_len; | ||||
reporter_data.dataLen = sizeof(int32_t); | reporter_data.dataLen = sizeof(int32_t); | ||||
GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", | |||||
this->Id()); | |||||
GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, | |||||
"Reporter data fail, model id:%u.", this->Id()); | |||||
reporter_data.data = (unsigned char *)fusion_op_name.c_str(); | reporter_data.data = (unsigned char *)fusion_op_name.c_str(); | ||||
reporter_data.dataLen = fusion_op_name_len; | reporter_data.dataLen = fusion_op_name_len; | ||||
GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", | |||||
this->Id()); | |||||
GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, | |||||
"Reporter data fail, model id:%u.", this->Id()); | |||||
// original op name before fusion | // original op name before fusion | ||||
reporter_data.data = (unsigned char *)&op_num; | reporter_data.data = (unsigned char *)&op_num; | ||||
reporter_data.dataLen = sizeof(int32_t); | reporter_data.dataLen = sizeof(int32_t); | ||||
GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", | |||||
this->Id()); | |||||
GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, | |||||
"Reporter data fail, model id:%u.", this->Id()); | |||||
for (uint32_t k = 0; k < op_num; k++) { | for (uint32_t k = 0; k < op_num; k++) { | ||||
std::string op_name = fusion_op_info->original_op_names[k]; | std::string op_name = fusion_op_info->original_op_names[k]; | ||||
int32_t op_name_len = op_name.size() == 0 ? 1 : op_name.size(); | int32_t op_name_len = op_name.size() == 0 ? 1 : op_name.size(); | ||||
reporter_data.data = (unsigned char *)&op_name_len; | reporter_data.data = (unsigned char *)&op_name_len; | ||||
reporter_data.dataLen = sizeof(int32_t); | reporter_data.dataLen = sizeof(int32_t); | ||||
GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", | |||||
this->Id()); | |||||
GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, | |||||
"Reporter data fail, model id:%u.", this->Id()); | |||||
reporter_data.data = (unsigned char *)op_name.c_str(); | reporter_data.data = (unsigned char *)op_name.c_str(); | ||||
reporter_data.dataLen = op_name_len; | reporter_data.dataLen = op_name_len; | ||||
GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", | |||||
this->Id()); | |||||
GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, | |||||
"Reporter data fail, model id:%u.", this->Id()); | |||||
} | } | ||||
// stream id info | // stream id info | ||||
uint32_t streamId = task->GetStreamId(); | uint32_t streamId = task->GetStreamId(); | ||||
reporter_data.data = (unsigned char *)&streamId; | reporter_data.data = (unsigned char *)&streamId; | ||||
reporter_data.dataLen = sizeof(int32_t); | reporter_data.dataLen = sizeof(int32_t); | ||||
GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", | |||||
this->Id()); | |||||
GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, | |||||
"Reporter data fail, model id:%u.", this->Id()); | |||||
// memory info | // memory info | ||||
struct memoryInfo memory_info; | struct memoryInfo memory_info; | ||||
@@ -2400,22 +2399,22 @@ Status DavinciModel::SinkModelProfile() { | |||||
memory_info.weight_size + memory_info.input_size + memory_info.output_size + memory_info.workspace_size; | memory_info.weight_size + memory_info.input_size + memory_info.output_size + memory_info.workspace_size; | ||||
reporter_data.data = (unsigned char *)&memory_info; | reporter_data.data = (unsigned char *)&memory_info; | ||||
reporter_data.dataLen = sizeof(struct memoryInfo); | reporter_data.dataLen = sizeof(struct memoryInfo); | ||||
GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", | |||||
this->Id()); | |||||
GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, | |||||
"Reporter data fail, model id:%u.", this->Id()); | |||||
// task info | // task info | ||||
reporter_data.data = (unsigned char *)&task_count; | reporter_data.data = (unsigned char *)&task_count; | ||||
reporter_data.dataLen = sizeof(uint32_t); | reporter_data.dataLen = sizeof(uint32_t); | ||||
GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", | |||||
this->Id()); | |||||
GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, | |||||
"Reporter data fail, model id:%u.", this->Id()); | |||||
Range task_range = op_id_map.equal_range(op_id); | Range task_range = op_id_map.equal_range(op_id); | ||||
for (CIT idx = task_range.first; idx != task_range.second; ++idx) { | for (CIT idx = task_range.first; idx != task_range.second; ++idx) { | ||||
uint32_t task_id = idx->second; | uint32_t task_id = idx->second; | ||||
reporter_data.data = (unsigned char *)&task_id; | reporter_data.data = (unsigned char *)&task_id; | ||||
reporter_data.dataLen = sizeof(uint32_t); | reporter_data.dataLen = sizeof(uint32_t); | ||||
GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", | |||||
this->Id()); | |||||
GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, | |||||
"Reporter data fail, model id:%u.", this->Id()); | |||||
} | } | ||||
} | } | ||||
} | } | ||||
@@ -2424,10 +2423,8 @@ Status DavinciModel::SinkModelProfile() { | |||||
Status DavinciModel::SinkTimeProfile(const InputData ¤t_data) { | Status DavinciModel::SinkTimeProfile(const InputData ¤t_data) { | ||||
// profiling plugin must be registered | // profiling plugin must be registered | ||||
Msprof::Engine::Reporter *reporter = PluginImpl::GetPluginReporter(); | |||||
GE_IF_BOOL_EXEC(reporter == nullptr, GELOGI("Profiling report is nullptr!"); return SUCCESS); | |||||
Msprof::Engine::ReporterData reporter_data{}; | |||||
auto &prof_mgr = ProfilingManager::Instance(); | |||||
ReporterData reporter_data{}; | |||||
// report model data tag name | // report model data tag name | ||||
std::string tag_name; | std::string tag_name; | ||||
tag_name.append("model_time_info_") | tag_name.append("model_time_info_") | ||||
@@ -2450,33 +2447,33 @@ Status DavinciModel::SinkTimeProfile(const InputData ¤t_data) { | |||||
size_t name_len = name.size(); | size_t name_len = name.size(); | ||||
reporter_data.data = (unsigned char *)&name_len; | reporter_data.data = (unsigned char *)&name_len; | ||||
reporter_data.dataLen = sizeof(int32_t); | reporter_data.dataLen = sizeof(int32_t); | ||||
GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", | |||||
this->Id()); | |||||
GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, | |||||
"Reporter data fail, model id:%u.", this->Id()); | |||||
reporter_data.data = (unsigned char *)name.c_str(); | reporter_data.data = (unsigned char *)name.c_str(); | ||||
reporter_data.dataLen = name.size(); | reporter_data.dataLen = name.size(); | ||||
GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", | |||||
this->Id()); | |||||
GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, | |||||
"Reporter data fail, model id:%u.", this->Id()); | |||||
// request id | // request id | ||||
uint64_t request_id = current_data.request_id; | uint64_t request_id = current_data.request_id; | ||||
reporter_data.data = (unsigned char *)&request_id; | reporter_data.data = (unsigned char *)&request_id; | ||||
reporter_data.dataLen = sizeof(uint32_t); | reporter_data.dataLen = sizeof(uint32_t); | ||||
GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, | |||||
GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, | |||||
"Reporter data fail, model id:%u, data index:%u.", this->Id(), current_data.index); | "Reporter data fail, model id:%u, data index:%u.", this->Id(), current_data.index); | ||||
// thread id | // thread id | ||||
int32_t thread_id = GetDataInputTid(); | int32_t thread_id = GetDataInputTid(); | ||||
reporter_data.data = (unsigned char *)&thread_id; | reporter_data.data = (unsigned char *)&thread_id; | ||||
reporter_data.dataLen = sizeof(int32_t); | reporter_data.dataLen = sizeof(int32_t); | ||||
GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, | |||||
GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, | |||||
"Reporter data fail, model id:%u, data index:%u.", this->Id(), current_data.index); | "Reporter data fail, model id:%u, data index:%u.", this->Id(), current_data.index); | ||||
// time info | // time info | ||||
time_info_.modelId = this->Id(); | time_info_.modelId = this->Id(); | ||||
reporter_data.data = (unsigned char *)&time_info_; | reporter_data.data = (unsigned char *)&time_info_; | ||||
reporter_data.dataLen = sizeof(struct timeInfo); | reporter_data.dataLen = sizeof(struct timeInfo); | ||||
GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, | |||||
GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, | |||||
"Reporter data fail, model id:%u, data index:%u.", this->Id(), current_data.index); | "Reporter data fail, model id:%u, data index:%u.", this->Id(), current_data.index); | ||||
return SUCCESS; | return SUCCESS; | ||||
@@ -2809,71 +2806,37 @@ void *DavinciModel::Run(DavinciModel *model) { | |||||
} | } | ||||
GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), model->SetProfileTime(MODEL_PRE_PROC_END)); | GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), model->SetProfileTime(MODEL_PRE_PROC_END)); | ||||
GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), model->SetProfileTime(MODEL_INFER_START)); | GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), model->SetProfileTime(MODEL_INFER_START)); | ||||
if (ProfilingManager::Instance().ProfilingOpTraceOn()) { | |||||
GELOGI("GetOpTraceIterNum:%d", ProfilingManager::Instance().GetOpTraceIterNum()); | |||||
for (int32_t i = 0; i < ProfilingManager::Instance().GetOpTraceIterNum(); i++) { | |||||
if (!ProfilingManager::Instance().ProfilingLoadFlag()) { | |||||
vector<int32_t> prof_device_id_vec = ProfilingManager::Instance().GetProfilingDeviceId(); | |||||
for (size_t j = 0; j < prof_device_id_vec.size(); ++j) { | |||||
// just profiling, no need to check value | |||||
(void)ProfilingManager::Instance().StartProfiling(i, prof_device_id_vec[j]); | |||||
} | |||||
} | |||||
GELOGI("rtModelExecute start."); | |||||
rt_ret = rtModelExecute(model->rt_model_handle_, model->rt_model_stream_, 0); | |||||
GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, rslt_flg = false; | |||||
(void)model->ReturnResult(current_data.index, false, false, data_wrapper->GetOutput()); | |||||
continue); // [No need to check value] | |||||
GELOGI("rtModelExecute end"); | |||||
GELOGI("rtStreamSynchronize start."); | |||||
rt_ret = rtStreamSynchronize(model->rt_model_stream_); | |||||
if (rt_ret == kModelAbortNormal || rt_ret == kModelAbortNormalNew) { | |||||
GELOGI("The model with multiple datasets aborts normally."); | |||||
} else { | |||||
GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, rslt_flg = false; | |||||
(void)model->ReturnResult(current_data.index, false, seq_end_flag, data_wrapper->GetOutput()); | |||||
continue); // [No need to check value] | |||||
} | |||||
GELOGI("rtStreamSynchronize end."); | |||||
(void)ProfilingManager::Instance().StopProfiling(); // just profiling, no need to check value | |||||
} | |||||
GE_TIMESTAMP_START(rtModelExecute); | |||||
GELOGI("rtModelExecute start."); | |||||
rt_ret = rtModelExecute(model->rt_model_handle_, model->rt_model_stream_, 0); | |||||
GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, rslt_flg = false; | |||||
(void)model->ReturnResult(current_data.index, false, false, data_wrapper->GetOutput()); | |||||
CsaInteract::GetInstance().WriteErrorCode(rt_ret, ERROR_MODULE_RUNTIME, JOBSUBSTATE_GRAPH_EXEC); | |||||
continue); | |||||
GELOGI("rtModelExecute end"); | |||||
GE_IF_BOOL_EXEC(model->is_first_execute_, GE_TIMESTAMP_EVENT_END(rtModelExecute, "GraphExcute::rtModelExecute")); | |||||
GE_TIMESTAMP_START(rtStreamSynchronize); | |||||
GELOGI("rtStreamSynchronize start."); | |||||
rt_ret = rtStreamSynchronize(model->rt_model_stream_); | |||||
if (rt_ret == kEndOfSequence || rt_ret == kEndOfSequenceNew) { | |||||
seq_end_flag = true; | |||||
} | |||||
if (rt_ret == kModelAbortNormal || rt_ret == kModelAbortNormalNew) { | |||||
GELOGI("The model with multiple datasets aborts normally."); | |||||
} else { | } else { | ||||
GE_TIMESTAMP_START(rtModelExecute); | |||||
GELOGI("rtModelExecute start."); | |||||
rt_ret = rtModelExecute(model->rt_model_handle_, model->rt_model_stream_, 0); | |||||
GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, rslt_flg = false; | |||||
(void)model->ReturnResult(current_data.index, false, false, data_wrapper->GetOutput()); | |||||
CsaInteract::GetInstance().WriteErrorCode(rt_ret, ERROR_MODULE_RUNTIME, JOBSUBSTATE_GRAPH_EXEC); | |||||
continue); | |||||
GELOGI("rtModelExecute end"); | |||||
GE_IF_BOOL_EXEC(model->is_first_execute_, GE_TIMESTAMP_EVENT_END(rtModelExecute, "GraphExcute::rtModelExecute")); | |||||
GE_TIMESTAMP_START(rtStreamSynchronize); | |||||
GELOGI("rtStreamSynchronize start."); | |||||
rt_ret = rtStreamSynchronize(model->rt_model_stream_); | |||||
if (rt_ret == kEndOfSequence || rt_ret == kEndOfSequenceNew) { | |||||
seq_end_flag = true; | |||||
} | |||||
if (rt_ret == kModelAbortNormal || rt_ret == kModelAbortNormalNew) { | |||||
GELOGI("The model with multiple datasets aborts normally."); | |||||
} else { | |||||
GE_IF_BOOL_EXEC( | |||||
rt_ret != RT_ERROR_NONE, rslt_flg = false; GELOGI("seq_end_flg: %d", seq_end_flag); | |||||
(void)model->ReturnResult(current_data.index, false, seq_end_flag, | |||||
data_wrapper->GetOutput()); // [No need to check value] | |||||
CsaInteract::GetInstance().StoreInternalErrorCode(rt_ret, ERROR_MODULE_RUNTIME, JOBSUBSTATE_GRAPH_EXEC); | |||||
continue); | |||||
} | |||||
GELOGI("rtStreamSynchronize end."); | |||||
GE_IF_BOOL_EXEC(model->is_first_execute_, | |||||
GE_TIMESTAMP_EVENT_END(rtStreamSynchronize, "GraphExcute::Wait for rtStreamSynchronize")); | |||||
GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), model->SetProfileTime(MODEL_INFER_END)); | |||||
GE_IF_BOOL_EXEC( | |||||
rt_ret != RT_ERROR_NONE, rslt_flg = false; GELOGI("seq_end_flg: %d", seq_end_flag); | |||||
(void)model->ReturnResult(current_data.index, false, seq_end_flag, | |||||
data_wrapper->GetOutput()); // [No need to check value] | |||||
CsaInteract::GetInstance().StoreInternalErrorCode(rt_ret, ERROR_MODULE_RUNTIME, JOBSUBSTATE_GRAPH_EXEC); | |||||
continue); | |||||
} | } | ||||
GELOGI("rtStreamSynchronize end."); | |||||
GE_IF_BOOL_EXEC(model->is_first_execute_, | |||||
GE_TIMESTAMP_EVENT_END(rtStreamSynchronize, "GraphExcute::Wait for rtStreamSynchronize")); | |||||
GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), model->SetProfileTime(MODEL_INFER_END)); | |||||
GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), | GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), | ||||
model->SetProfileTime(MODEL_AFTER_PROC_START)); | model->SetProfileTime(MODEL_AFTER_PROC_START)); | ||||
GE_TIMESTAMP_START(ReturnResult3); | GE_TIMESTAMP_START(ReturnResult3); | ||||
@@ -440,7 +440,7 @@ class DavinciModel { | |||||
Status SinkTimeProfile(const InputData ¤t_data); | Status SinkTimeProfile(const InputData ¤t_data); | ||||
Status ReportProfilingData(bool check_device = true); | |||||
Status ReportProfilingData(); | |||||
void SaveDumpOpInfo(const RuntimeParam &model_param, const OpDescPtr &op, uint32_t task_id, uint32_t stream_id) { | void SaveDumpOpInfo(const RuntimeParam &model_param, const OpDescPtr &op, uint32_t task_id, uint32_t stream_id) { | ||||
data_dumper_.SaveDumpOpInfo(model_param, op, task_id, stream_id); | data_dumper_.SaveDumpOpInfo(model_param, op, task_id, stream_id); | ||||
@@ -40,9 +40,7 @@ const int kCmdParSize = 2; | |||||
const int kDumpCmdPairSize = 2; | const int kDumpCmdPairSize = 2; | ||||
const std::size_t kProfCmdParaMaxSize = 1000; | const std::size_t kProfCmdParaMaxSize = 1000; | ||||
const std::size_t kProfStartCmdParaSize = 2; | const std::size_t kProfStartCmdParaSize = 2; | ||||
const std::string kCmdTypeProfile = "profile"; | |||||
const std::string kCmdTypeDump = "dump"; | const std::string kCmdTypeDump = "dump"; | ||||
const std::string kCmdTypeProfiling = "profiling"; | |||||
const std::string kCmdTypeProfInit = "prof_init"; | const std::string kCmdTypeProfInit = "prof_init"; | ||||
const std::string kCmdTypeProfFinalize = "prof_finalize"; | const std::string kCmdTypeProfFinalize = "prof_finalize"; | ||||
const std::string kCmdTypeProfStart = "prof_start"; | const std::string kCmdTypeProfStart = "prof_start"; | ||||
@@ -632,8 +630,7 @@ Status ModelManager::Stop(uint32_t model_id) { | |||||
/// | /// | ||||
Status ModelManager::HandleCommand(const Command &command) { | Status ModelManager::HandleCommand(const Command &command) { | ||||
static const std::map<std::string, std::function<uint32_t(const Command &)>> cmds = { | static const std::map<std::string, std::function<uint32_t(const Command &)>> cmds = { | ||||
{kCmdTypeProfile, HandleProfileCommand}, {kCmdTypeDump, HandleDumpCommand}, | |||||
{kCmdTypeProfiling, HandleAclProfilingCommand}, {kCmdTypeProfInit, HandleProfInitCommand}, | |||||
{kCmdTypeDump, HandleDumpCommand}, {kCmdTypeProfInit, HandleProfInitCommand}, | |||||
{kCmdTypeProfFinalize, HandleProfFinalizeCommand}, {kCmdTypeProfStart, HandleProfStartCommand}, | {kCmdTypeProfFinalize, HandleProfFinalizeCommand}, {kCmdTypeProfStart, HandleProfStartCommand}, | ||||
{kCmdTypeProfStop, HandleProfStopCommand}, | {kCmdTypeProfStop, HandleProfStopCommand}, | ||||
{kCmdTypeProfModelSubscribe, HandleProfModelSubscribeCommand}, | {kCmdTypeProfModelSubscribe, HandleProfModelSubscribeCommand}, | ||||
@@ -648,21 +645,6 @@ Status ModelManager::HandleCommand(const Command &command) { | |||||
} | } | ||||
} | } | ||||
Status ModelManager::HandleAclProfilingCommand(const Command &command) { | |||||
if (command.cmd_params.size() < kCmdParSize) { | |||||
GELOGE(PARAM_INVALID, "When the cmd_type is 'profiling', the size of cmd_params must larger than 2."); | |||||
return PARAM_INVALID; | |||||
} | |||||
std::string map_key = command.cmd_params[0]; | |||||
std::string value = command.cmd_params[1]; | |||||
if (map_key == PROFILE_CONFIG) { | |||||
ProfilingManager::Instance().SetProfilingConfig(value); | |||||
} | |||||
return SUCCESS; | |||||
} | |||||
Status ModelManager::GetModelByCmd(const Command &command, | Status ModelManager::GetModelByCmd(const Command &command, | ||||
std::shared_ptr<DavinciModel> &davinci_model) { | std::shared_ptr<DavinciModel> &davinci_model) { | ||||
if (command.cmd_params.size() < kCmdParSize) { | if (command.cmd_params.size() < kCmdParSize) { | ||||
@@ -809,29 +791,6 @@ Status ModelManager::HandleProfStopCommand(const Command &command) { | |||||
return SUCCESS; | return SUCCESS; | ||||
} | } | ||||
Status ModelManager::HandleProfileCommand(const Command &command) { | |||||
if (command.cmd_params.size() < kCmdParSize) { | |||||
GELOGE(PARAM_INVALID, "When the cmd_type is 'profile', the size of cmd_params must larger than 2."); | |||||
return PARAM_INVALID; | |||||
} | |||||
std::string map_key = command.cmd_params[0]; | |||||
std::string value = command.cmd_params[1]; | |||||
GELOGI("Profiling mode, Command key:%s , value:%s ", map_key.c_str(), value.c_str()); | |||||
auto iter = PROFILE_COMPONENT_MAP.find(map_key); | |||||
if (iter != PROFILE_COMPONENT_MAP.end()) { | |||||
std::string property_value = (value == "on") ? "1" : "0"; | |||||
PropertiesManager::Instance().SetPropertyValue(iter->second, property_value); | |||||
} | |||||
if ((map_key == PROFILER_JOBCTX || map_key == PROFILER_TARGET_PATH || map_key == RTS_PROFILE_PATH)) { | |||||
PropertiesManager::Instance().SetPropertyValue(map_key, value); | |||||
} | |||||
return SUCCESS; | |||||
} | |||||
static Status ParserPara(const Command &command, const string &dump_key, string &dump_value) { | static Status ParserPara(const Command &command, const string &dump_key, string &dump_value) { | ||||
auto iter = std::find(command.cmd_params.begin(), command.cmd_params.end(), dump_key); | auto iter = std::find(command.cmd_params.begin(), command.cmd_params.end(), dump_key); | ||||
if (iter != command.cmd_params.end()) { | if (iter != command.cmd_params.end()) { | ||||
@@ -169,8 +169,6 @@ class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ModelManager { | |||||
/// @brief comment handle function | /// @brief comment handle function | ||||
/// | /// | ||||
ge::Status HandleCommand(const Command &command); | ge::Status HandleCommand(const Command &command); | ||||
static ge::Status HandleAclProfilingCommand(const Command &command); | |||||
static ge::Status HandleProfileCommand(const Command &command); | |||||
static ge::Status HandleDumpCommand(const Command &command); | static ge::Status HandleDumpCommand(const Command &command); | ||||
static ge::Status HandleProfModelSubscribeCommand(const Command &command); | static ge::Status HandleProfModelSubscribeCommand(const Command &command); | ||||
static ge::Status HandleProfModelUnsubscribeCommand(const Command &command); | static ge::Status HandleProfModelUnsubscribeCommand(const Command &command); | ||||
@@ -260,8 +260,7 @@ Status NodeDoneCallback::ProfilingReport() { | |||||
} | } | ||||
auto &profiling_manager = ProfilingManager::Instance(); | auto &profiling_manager = ProfilingManager::Instance(); | ||||
profiling_manager.ReportProfilingData(model->GetModelId(), task_desc_info, compute_graph_info, | |||||
!profiling_manager.IsAclApiMode()); | |||||
profiling_manager.ReportProfilingData(model->GetModelId(), task_desc_info, compute_graph_info); | |||||
return SUCCESS; | return SUCCESS; | ||||
} | } | ||||
@@ -485,11 +485,9 @@ Status GELib::Finalize() { | |||||
void GELib::ShutDownProfiling() { | void GELib::ShutDownProfiling() { | ||||
std::lock_guard<std::mutex> lock(status_mutex_); | std::lock_guard<std::mutex> lock(status_mutex_); | ||||
if (!ProfilingManager::Instance().ProfilingOpTraceOn() && ProfilingManager::Instance().ProfilingOn()) { | |||||
ProfilingManager::Instance().StopProfiling(); | |||||
} | |||||
if (ProfilingManager::Instance().ProfilingOn()) { | if (ProfilingManager::Instance().ProfilingOn()) { | ||||
ProfilingManager::Instance().PluginUnInit(GE_PROFILING_MODULE); | |||||
ProfilingManager::Instance().StopProfiling(); | |||||
ProfilingManager::Instance().PluginUnInit(); | |||||
} | } | ||||
} | } | ||||
@@ -167,4 +167,5 @@ Status OpsKernelBuilderManager::GenerateTask(const Node &node, | |||||
GELOGD("Done invoking GenerateTask successfully"); | GELOGD("Done invoking GenerateTask successfully"); | ||||
return SUCCESS; | return SUCCESS; | ||||
} | } | ||||
} // namespace ge | |||||
} // namespace ge |
@@ -72,8 +72,7 @@ Status ProfilingTaskInfo(OpTask *op_task) { | |||||
std::vector<ComputeGraphDescInfo> compute_graph_info; | std::vector<ComputeGraphDescInfo> compute_graph_info; | ||||
auto &profiling_manager = ProfilingManager::Instance(); | auto &profiling_manager = ProfilingManager::Instance(); | ||||
profiling_manager.ReportProfilingData(model_id, task_desc_info, compute_graph_info, | |||||
!profiling_manager.IsAclApiMode()); | |||||
profiling_manager.ReportProfilingData(model_id, task_desc_info, compute_graph_info); | |||||
return SUCCESS; | return SUCCESS; | ||||
} | } | ||||
} // namespace | } // namespace | ||||
@@ -71,7 +71,7 @@ max_code_len_per_line = 100 | |||||
when DEBUG on | when DEBUG on | ||||
""" | """ | ||||
white_list_for_debug = ["attr_value.h", "operator.h", "tensor.h", "graph.h", "operator_factory.h", | white_list_for_debug = ["attr_value.h", "operator.h", "tensor.h", "graph.h", "operator_factory.h", | ||||
"ge_ir_build.h", "ge_api.h", "ge_prof.h", "tensorflow_parser.h", "caffe_parser.h"] | |||||
"ge_ir_build.h", "ge_api.h", "tensorflow_parser.h", "caffe_parser.h"] | |||||
include_dir_key_words = ["ge", "graph", "parser"] | include_dir_key_words = ["ge", "graph", "parser"] | ||||
DEBUG = True | DEBUG = True | ||||
@@ -1,102 +0,0 @@ | |||||
/** | |||||
* Copyright 2019-2020 Huawei Technologies Co., Ltd | |||||
* | |||||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||||
* you may not use this file except in compliance with the License. | |||||
* You may obtain a copy of the License at | |||||
* | |||||
* http://www.apache.org/licenses/LICENSE-2.0 | |||||
* | |||||
* Unless required by applicable law or agreed to in writing, software | |||||
* distributed under the License is distributed on an "AS IS" BASIS, | |||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
* See the License for the specific language governing permissions and | |||||
* limitations under the License. | |||||
*/ | |||||
#ifndef INC_EXTERNAL_GE_GE_PROF_H_ | |||||
#define INC_EXTERNAL_GE_GE_PROF_H_ | |||||
#include <map> | |||||
#include <string> | |||||
#include <vector> | |||||
#include "ge/ge_api_error_codes.h" | |||||
namespace ge { | |||||
enum ProfDataTypeConfig { | |||||
kProfTaskTime = 0x0002, | |||||
kProfAiCoreMetrics = 0x0004, | |||||
kProfAicpuTrace = 0x0008, | |||||
kProfTrainingTrace = 0x0800, | |||||
kProfHcclTrace = 0x1000 | |||||
}; | |||||
enum ProfilingAicoreMetrics { | |||||
kAicoreArithmaticThroughput = 0, | |||||
kAicorePipeline = 1, | |||||
kAicoreSynchronization = 2, | |||||
kAicoreMemory = 3, | |||||
kAicoreInternalMemory = 4, | |||||
kAicoreStall = 5 | |||||
}; | |||||
typedef struct ProfAicoreEvents ProfAicoreEvents; | |||||
typedef struct aclgrphProfConfig aclgrphProfConfig; | |||||
/// | |||||
/// @ingroup AscendCL | |||||
/// @brief Initialize the profiling and set profiling configuration path | |||||
/// @param [in] profiler_path: configuration path of profiling | |||||
/// @param [in] length: length of configuration path | |||||
/// @return Status result of function | |||||
/// | |||||
Status aclgrphProfInit(const char *profiler_path, uint32_t length); | |||||
/// | |||||
/// @ingroup AscendCL | |||||
/// @brief Finalize profiling | |||||
/// @return Status result of function | |||||
/// | |||||
Status aclgrphProfFinalize(); | |||||
/// | |||||
/// @ingroup AscendCL | |||||
/// @brief Create data of type aclgrphProfConfig | |||||
/// @param [in] deviceid_list: device id list | |||||
/// @param [in] device_nums: device numbers | |||||
/// @param [in] aicore_metrics: type of aicore metrics | |||||
/// @param [in] aicore_events: pointer to aicore events be reserved, only support NULL now | |||||
/// @param [in] data_type_config: modules need profiling | |||||
/// @return Status result of function | |||||
/// | |||||
aclgrphProfConfig *aclgrphProfCreateConfig(uint32_t *deviceid_list, uint32_t device_nums, | |||||
ProfilingAicoreMetrics aicore_metrics, ProfAicoreEvents *aicore_events, | |||||
uint64_t data_type_config); | |||||
/// | |||||
/// @ingroup AscendCL | |||||
/// @brief Destroy data of type aclgrphProfConfig | |||||
/// @param [in] profiler_config: config of profiling | |||||
/// @return Status result of function | |||||
/// | |||||
Status aclgrphProfDestroyConfig(aclgrphProfConfig *profiler_config); | |||||
/// | |||||
/// @ingroup AscendCL | |||||
/// @brief Start profiling of modules which is configured by profiler config | |||||
/// @param [in] profiler_config: config of profiling | |||||
/// @return Status result of function | |||||
/// | |||||
Status aclgrphProfStart(aclgrphProfConfig *profiler_config); | |||||
/// | |||||
/// @ingroup AscendCL | |||||
/// @brief Stop profiling of modules which is configured by profiler config | |||||
/// @param [in] profiler_config: config of profiling | |||||
/// @return Status result of function | |||||
/// | |||||
Status aclgrphProfStop(aclgrphProfConfig *profiler_config); | |||||
} // namespace ge | |||||
#endif // INC_EXTERNAL_GE_GE_PROF_H_ |
@@ -0,0 +1,45 @@ | |||||
/** | |||||
* Copyright 2020 Huawei Technologies Co., Ltd | |||||
* | |||||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||||
* you may not use this file except in compliance with the License. | |||||
* You may obtain a copy of the License at | |||||
* | |||||
* http://www.apache.org/licenses/LICENSE-2.0 | |||||
* | |||||
* Unless required by applicable law or agreed to in writing, software | |||||
* distributed under the License is distributed on an "AS IS" BASIS, | |||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
* See the License for the specific language governing permissions and | |||||
* limitations under the License. | |||||
*/ | |||||
#ifndef INC_FRAMEWORK_COMMON_GE_PROFILING_H_ | |||||
#define INC_FRAMEWORK_COMMON_GE_PROFILING_H_ | |||||
#include "ge/ge_api_error_codes.h" | |||||
#include "toolchain/prof_callback.h" | |||||
#define MAX_DEV_NUM (64) | |||||
enum ProfCommandHandleType { | |||||
kProfCommandhandleInit = 0, | |||||
kProfCommandhandleStart, | |||||
kProfCommandhandleStop, | |||||
kProfCommandhandleFinalize, | |||||
kProfCommandhandleModelSubscribe, | |||||
kProfCommandhandleModelUnsubscribe | |||||
}; | |||||
struct ProfCommandHandleData { | |||||
uint64_t profSwitch; | |||||
uint32_t devNums; // length of device id list | |||||
uint32_t devIdList[MAX_DEV_NUM]; | |||||
uint32_t modelId; | |||||
}; | |||||
ge::Status RegProfCtrlCallback(MsprofCtrlCallback func); | |||||
ge::Status RegProfSetDeviceCallback(MsprofSetDeviceCallback func); | |||||
ge::Status RegProfReporterCallback(MsprofReporterCallback func); | |||||
ge::Status ProfCommandHandle(ProfCommandHandleType type, void *data, uint32_t len); | |||||
#endif // INC_FRAMEWORK_COMMON_GE_PROFILING_H_ |
@@ -0,0 +1,24 @@ | |||||
/** | |||||
* Copyright 2020 Huawei Technologies Co., Ltd | |||||
* | |||||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||||
* you may not use this file except in compliance with the License. | |||||
* You may obtain a copy of the License at | |||||
* | |||||
* http://www.apache.org/licenses/LICENSE-2.0 | |||||
* | |||||
* Unless required by applicable law or agreed to in writing, software | |||||
* distributed under the License is distributed on an "AS IS" BASIS, | |||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||||
* See the License for the specific language governing permissions and | |||||
* limitations under the License. | |||||
*/ | |||||
#ifndef INC_FRAMEWORK_COMMON_GE_RUNNER_PROFILING_H_ | |||||
#define INC_FRAMEWORK_COMMON_GE_RUNNER_PROFILING_H_ | |||||
#include "profiling/ge_profiling.h" | |||||
bool IsInitialize(); | |||||
#endif // INC_FRAMEWORK_COMMON_GE_RUNNER_PROFILING_H_ |
@@ -1 +1 @@ | |||||
Subproject commit c85822cd5404e40cb4ff2bfc9483062648c13c57 | |||||
Subproject commit dba83744a3ffe3d5f89496e69bb65c50f800c299 |
@@ -1 +1 @@ | |||||
Subproject commit 5bc8c38b37476e8f4b9391c96e4a2cca59e53d8e | |||||
Subproject commit ce574894f13cd94749d1a3964a13e8c97c20434a |