@@ -71,7 +71,7 @@ if (ENABLE_OPEN_SRC) | |||
set(STATIC_ACL_LIB ${GE_LIB_PATH}) | |||
find_module(slog libslog.so ${GE_LIB_PATH}) | |||
find_module(static_mmpa libmmpa.a ${GE_LIB_PATH}) | |||
find_module(msprof libmsprof.so ${GE_LIB_PATH}) | |||
find_module(msprofiler libmsprofiler.a ${GE_LIB_PATH}) | |||
find_module(hccl libhccl.so ${GE_LIB_PATH}) | |||
find_module(adump_server libadump_server.a ${GE_LIB_PATH}) | |||
find_module(runtime libruntime.so ${GE_LIB_PATH}) | |||
@@ -80,20 +80,19 @@ if (ENABLE_OPEN_SRC) | |||
find_module(error_manager liberror_manager.so ${GE_LIB_PATH}) | |||
find_module(ascend_hal_stub libascend_hal.so ${GE_LIB_PATH}) | |||
find_module(error_manager_static liberror_manager.a ${GE_LIB_PATH}) | |||
find_module(msprofiler libmsprofiler.a ${GE_LIB_PATH}) | |||
find_module(msprofiler_fwk libmsprofiler_fwk.a ${GE_LIB_PATH}) | |||
#find_module(ascendcl_static libascendcl.a ${GE_LIB_PATH}) | |||
else() | |||
find_module(slog libslog.so ${ASCEND_ATC_DIR}) | |||
find_module(static_mmpa libmmpa.a ${ASCEND_ATC_DIR}) | |||
find_module(error_manager liberror_manager.so ${ASCEND_ATC_DIR}) | |||
if(PLATFORM STREQUAL "train") | |||
find_module(msprof libmsprof.so ${ASCEND_DRIVER_COMMON_DIR}) | |||
find_module(hccl libhccl.so ${ASCEND_RUNTIME_DIR}) | |||
find_module(adump_server libadump_server.a ${ASCEND_RUNTIME_DIR}) | |||
find_module(runtime libruntime.so ${ASCEND_RUNTIME_DIR}) | |||
find_module(resource libresource.so ${ASCEND_RUNTIME_DIR}) | |||
find_module(error_manager liberror_manager.so ${ASCEND_RUNTIME_DIR}) | |||
find_module(msprofiler libmsprofiler.a ${ASCEND_RUNTIME_DIR}) | |||
find_module(msprofiler_fwk libmsprofiler_fwk.a ${ASCEND_RUNTIME_DIR}) | |||
find_module(ascend_hal_stub libascend_hal.so ${ASCEND_DRIVER_DIR}/driver) | |||
if(PRODUCT STREQUAL "flr3") | |||
message(FATAL_ERROR "This platform is not supported in train mode, build terminated") | |||
@@ -108,18 +107,15 @@ if (ENABLE_OPEN_SRC) | |||
find_module(msprofiler libmsprofiler.a ${ASCEND_ACL_DIR}) | |||
#find_module(ascendcl_static libascendcl.a ${ASCEND_ACL_DIR}) | |||
if(PRODUCT STREQUAL "flr3") | |||
find_module(msprof libmsprof.so ${ASCEND_DRIVER_SHARE_DIR}) | |||
elseif(PRODUCT STREQUAL "flr1") | |||
find_module(ascend_hal_stub libascend_hal.so ${ASCEND_DRIVER_DIR}/driver) | |||
find_module(msprof libmsprof.so ${ASCEND_DRIVER_COMMON_DIR}) | |||
elseif(PRODUCT STREQUAL "flr2") | |||
# flr2 ascend_hal_stub limsprof ? | |||
else() | |||
find_module(ascend_hal_stub libascend_hal.so ${ASCEND_DRIVER_DIR}) | |||
find_module(msprof libmsprof.so ${ASCEND_DRIVER_DIR}) | |||
endif() | |||
elseif(PLATFORM STREQUAL "all") | |||
find_module(msprof libmsprof.so ${ASCEND_DRIVER_COMMON_DIR}) | |||
find_module(msprofiler libmsprofiler.a ${ASCEND_DRIVER_COMMON_DIR}) | |||
find_module(hccl libhccl.so ${ASCEND_RUNTIME_DIR}) | |||
find_module(adump_server libadump_server.a ${ASCEND_ACL_DIR}) | |||
find_module(runtime libruntime.so ${ASCEND_ACL_DIR}) | |||
@@ -127,7 +123,7 @@ if (ENABLE_OPEN_SRC) | |||
find_module(resource libresource.so ${ASCEND_ATC_DIR}) | |||
find_module(error_manager liberror_manager.so ${ASCEND_ATC_DIR}) | |||
find_module(error_manager_static liberror_manager.a ${ASCEND_ACL_DIR}) | |||
find_module(msprofiler libmsprofiler.a ${ASCEND_ACL_DIR}) | |||
find_module(msprofiler_fwk libmsprofiler_fwk.a ${ASCEND_ACL_DIR}) | |||
find_module(ascend_hal_stub libascend_hal.so ${ASCEND_DRIVER_DIR}/driver) | |||
#find_module(ascendcl_static libascendcl.a ${ASCEND_ACL_DIR}) | |||
else() | |||
@@ -60,6 +60,8 @@ set(TRAIN_SRC_LIST | |||
"common/dump/dump_manager.cc" | |||
"common/dump/dump_properties.cc" | |||
"common/dump/dump_op.cc" | |||
"common/profiling/ge_profiling.cc" | |||
"common/profiling/ge_runner_profiling.cc" | |||
"engine_manager/dnnengine_manager.cc" | |||
"ge_local_engine/engine/host_cpu_engine.cc" | |||
"generator/ge_generator.cc" | |||
@@ -332,7 +334,6 @@ set(TRAIN_SRC_LIST | |||
"hybrid/hybrid_davinci_model.cc" | |||
"executor/ge_executor.cc" | |||
"client/ge_api.cc" | |||
"client/ge_prof.cc" | |||
"analyzer/analyzer.cc" | |||
"ir_build/ge_ir_build.cc" | |||
"ir_build/atc_ir_common.cc" | |||
@@ -604,7 +605,7 @@ set(INFER_SRC_LIST | |||
if (NOT ENABLE_D AND NOT ENABLE_ACL AND NOT ENABLE_MS_TESTCASES) | |||
############ libge_runner.so ############ | |||
add_library(ge_runner SHARED ${TRAIN_SRC_LIST} ${PROTO_SRCS} ${PROTO_CLIENT_SRCS}) | |||
add_library(ge_runner SHARED ${TRAIN_SRC_LIST} ${PROTO_SRCS} ${PROTO_CLIENT_SRCS} $<TARGET_OBJECTS:msprofiler_fwk>) | |||
target_compile_definitions(ge_runner PRIVATE | |||
PROTOBUF_INLINE_NOT_IN_HEADERS=0 | |||
@@ -649,7 +650,6 @@ target_link_libraries(ge_runner | |||
$<BUILD_INTERFACE:intf_pub> | |||
ge_memory | |||
adump_server | |||
msprofiler | |||
static_mmpa | |||
-Wl,--no-as-needed | |||
graph | |||
@@ -658,7 +658,6 @@ target_link_libraries(ge_runner | |||
register | |||
c_sec | |||
slog | |||
msprof | |||
runtime | |||
resource | |||
error_manager | |||
@@ -783,7 +782,6 @@ target_link_libraries(opensrc_ascendcl PRIVATE | |||
c_sec | |||
runtime | |||
slog | |||
msprof | |||
ascend_hal_stub | |||
-Wl,--as-needed | |||
-lrt | |||
@@ -799,12 +797,10 @@ set_target_properties(opensrc_ascendcl PROPERTIES | |||
add_custom_command( | |||
OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/stub_ge_ir_build.cc | |||
${CMAKE_CURRENT_BINARY_DIR}/stub_ge_api.cc | |||
${CMAKE_CURRENT_BINARY_DIR}/stub_ge_prof.cc | |||
COMMAND echo "Generating stub files." | |||
&& ${HI_PYTHON} ${CMAKE_CURRENT_LIST_DIR}/stub/gen_stubapi.py ${GE_CODE_DIR}/inc/external ${CMAKE_CURRENT_BINARY_DIR} | |||
&& mv ge_ir_build.cc stub_ge_ir_build.cc | |||
&& mv ge_api.cc stub_ge_api.cc | |||
&& mv ge_prof.cc stub_ge_prof.cc | |||
&& echo "Generating stub files end." | |||
#WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} | |||
#DEPENDS stub/gen_stubapi.py ${TOP_DIR}/inc/external ${CMAKE_CURRENT_BINARY_DIR} | |||
@@ -813,7 +809,6 @@ add_custom_command( | |||
add_custom_target(ge_stub | |||
DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/stub_ge_ir_build.cc | |||
${CMAKE_CURRENT_BINARY_DIR}/stub_ge_api.cc | |||
${CMAKE_CURRENT_BINARY_DIR}/stub_ge_prof.cc | |||
) | |||
################################################################## | |||
@@ -855,7 +850,6 @@ target_include_directories(atc_stub_ge_compiler PRIVATE | |||
############ stub/libge_runner.so ############ | |||
add_library(fwk_stub_ge_runner SHARED | |||
stub_ge_api.cc | |||
stub_ge_prof.cc | |||
stub_ge_ir_build.cc | |||
) | |||
@@ -1,369 +0,0 @@ | |||
/** | |||
* Copyright 2020 Huawei Technologies Co., Ltd | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
*/ | |||
#include "ge/ge_prof.h" | |||
#include "ge/ge_api.h" | |||
#include "init/gelib.h" | |||
#include "common/debug/log.h" | |||
#include "framework/common/debug/ge_log.h" | |||
#include "common/profiling/profiling_manager.h" | |||
#include "graph/load/graph_loader.h" | |||
#include "toolchain/prof_acl_api.h" | |||
using std::map; | |||
using std::string; | |||
using std::vector; | |||
namespace { | |||
const uint32_t kMaxDeviceNum = 64; | |||
const uint32_t kDeviceListIndex = 3; | |||
const std::string kProfilingInit = "prof_init"; | |||
const std::string kProfilingFinalize = "prof_finalize"; | |||
const std::string kProfilingStart = "prof_start"; | |||
const std::string kProfilingStop = "prof_stop"; | |||
const std::string kDeviceNums = "devNums"; | |||
const std::string kDeviceIdList = "devIdList"; | |||
const std::string kAicoreMetrics = "aicoreMetrics"; | |||
const std::map<ge::ProfilingAicoreMetrics, std::string> kProfAicoreMetricsToString = { | |||
{ge::kAicoreArithmaticThroughput, "AICORE_ARITHMATIC_THROUGHPUT"}, | |||
{ge::kAicorePipeline, "AICORE_PIPELINE"}, | |||
{ge::kAicoreSynchronization, "AICORE_SYNCHRONIZATION"}, | |||
{ge::kAicoreMemory, "AICORE_MEMORY"}, | |||
{ge::kAicoreInternalMemory, "AICORE_INTERNAL_MEMORY"}, | |||
{ge::kAicoreStall, "AICORE_STALL"}}; | |||
} // namespace | |||
static bool g_graph_prof_init_ = false; | |||
static std::mutex g_prof_mutex_; | |||
namespace ge { | |||
struct aclgrphProfConfig { | |||
ProfConfig config; | |||
}; | |||
Status aclgrphProfInit(const char *profiler_path, uint32_t length) { | |||
GELOGT(TRACE_INIT, "Graph prof init start"); | |||
std::shared_ptr<GELib> instance_ptr = ge::GELib::GetInstance(); | |||
if (instance_ptr == nullptr || !instance_ptr->InitFlag()) { | |||
GELOGE(GE_CLI_GE_NOT_INITIALIZED, "Ge client is not initialized."); | |||
return FAILED; | |||
} | |||
std::lock_guard<std::mutex> lock(g_prof_mutex_); | |||
if (g_graph_prof_init_) { | |||
GELOGW("Multi graph profiling initializations."); | |||
return GE_PROF_MULTI_INIT; | |||
} | |||
Status ret = CheckPath(profiler_path, length); | |||
if (ret != SUCCESS) { | |||
GELOGE(ret, "Profiling config path is invalid."); | |||
return ret; | |||
} | |||
// if command mode is set, just return | |||
if (ProfilingManager::Instance().ProfilingOn()) { | |||
GELOGW("Graph prof init failed, cause profiling command pattern is running."); | |||
return GE_PROF_MODE_CONFLICT; | |||
} | |||
ret = ProfInit(profiler_path); | |||
if (ret != SUCCESS) { | |||
GELOGE(ret, "ProfInit init fail"); | |||
return ret; | |||
} | |||
GraphLoader graph_loader; | |||
Command command; | |||
command.cmd_params.clear(); | |||
command.cmd_type = kProfilingInit; | |||
command.module_index = PROF_MODEL_LOAD; | |||
ret = graph_loader.CommandHandle(command); | |||
if (ret != SUCCESS) { | |||
GELOGE(ret, "Handle profiling command %s failed, config = %s", kProfilingInit.c_str(), profiler_path); | |||
return ret; | |||
} | |||
if (!g_graph_prof_init_) { | |||
g_graph_prof_init_ = true; | |||
GELOGI("Profiling init successfully."); | |||
} | |||
GELOGI("Successfully execute GraphProfInit."); | |||
return SUCCESS; | |||
} | |||
Status aclgrphProfFinalize() { | |||
std::shared_ptr<GELib> instance_ptr = ge::GELib::GetInstance(); | |||
if (instance_ptr == nullptr || !instance_ptr->InitFlag()) { | |||
GELOGE(GE_CLI_GE_NOT_INITIALIZED, "Ge client is not initialized."); | |||
return FAILED; | |||
} | |||
std::lock_guard<std::mutex> lock(g_prof_mutex_); | |||
// if command mode is set, just return | |||
if (ProfilingManager::Instance().ProfilingOn()) { | |||
GELOGW("Graph prof finalize failed, cause profiling command pattern is running."); | |||
return GE_PROF_MODE_CONFLICT; | |||
} | |||
if (!g_graph_prof_init_) { | |||
GELOGE(GE_PROF_NOT_INIT, "Graph not profiling initialize."); | |||
return GE_PROF_NOT_INIT; | |||
} | |||
GraphLoader graph_loader; | |||
Command command; | |||
command.cmd_params.clear(); | |||
command.cmd_type = kProfilingFinalize; | |||
Status ret = graph_loader.CommandHandle(command); | |||
if (ret != SUCCESS) { | |||
GELOGE(ret, "Handle profiling command %s failed.", kProfilingFinalize.c_str()); | |||
return ret; | |||
} | |||
ret = ProfFinalize(); | |||
if (ret != SUCCESS) { | |||
GELOGE(ret, "Finalize profiling failed, result = %d", ret); | |||
} | |||
if (ret == SUCCESS) { | |||
g_graph_prof_init_ = false; | |||
GELOGI("Successfully execute GraphProfFinalize."); | |||
} | |||
return ret; | |||
} | |||
bool TransProfConfigToParam(const aclgrphProfConfig *profiler_config, vector<string> &prof_config_params) { | |||
prof_config_params.clear(); | |||
prof_config_params.emplace_back(kDeviceNums); | |||
prof_config_params.emplace_back(std::to_string(profiler_config->config.devNums)); | |||
prof_config_params.emplace_back(kDeviceIdList); | |||
std::string devID = ""; | |||
if (profiler_config->config.devNums == 0) { | |||
GELOGW("The device num is invalid."); | |||
return false; | |||
} | |||
for (uint32_t i = 0; i < profiler_config->config.devNums; i++) { | |||
devID.append(std::to_string(profiler_config->config.devIdList[i])); | |||
if (i != profiler_config->config.devNums - 1) { | |||
devID.append(","); | |||
} | |||
} | |||
prof_config_params.push_back(devID); | |||
prof_config_params.push_back(kAicoreMetrics); | |||
auto iter = | |||
kProfAicoreMetricsToString.find(static_cast<ProfilingAicoreMetrics>(profiler_config->config.aicoreMetrics)); | |||
if (iter == kProfAicoreMetricsToString.end()) { | |||
GELOGW("The prof aicore metrics is invalid."); | |||
return false; | |||
} | |||
prof_config_params.push_back(iter->second); | |||
return true; | |||
} | |||
bool isProfConfigValid(const uint32_t *deviceid_list, uint32_t device_nums) { | |||
if (deviceid_list == nullptr) { | |||
GELOGE(PARAM_INVALID, "deviceIdList is nullptr"); | |||
return false; | |||
} | |||
if (device_nums == 0 || device_nums > kMaxDeviceNum) { | |||
GELOGE(PARAM_INVALID, "The device nums is invalid."); | |||
return false; | |||
} | |||
// real device num | |||
int32_t dev_count = 0; | |||
rtError_t rt_err = rtGetDeviceCount(&dev_count); | |||
if (rt_err != RT_ERROR_NONE) { | |||
GELOGE(INTERNAL_ERROR, "Get the Device count fail."); | |||
return false; | |||
} | |||
if (device_nums > static_cast<uint32_t>(dev_count)) { | |||
GELOGE(PARAM_INVALID, "Device num(%u) is not in range 1 ~ %d.", device_nums, dev_count); | |||
return false; | |||
} | |||
std::unordered_set<uint32_t> record; | |||
for (size_t i = 0; i < device_nums; ++i) { | |||
uint32_t dev_id = deviceid_list[i]; | |||
if (dev_id >= static_cast<uint32_t>(dev_count)) { | |||
GELOGE(PARAM_INVALID, "Device id %u is not in range 0 ~ %d(exclude %d)", dev_id, dev_count, dev_count); | |||
return false; | |||
} | |||
if (record.count(dev_id) > 0) { | |||
GELOGE(PARAM_INVALID, "Device id %u is duplicatedly set", dev_id); | |||
return false; | |||
} | |||
record.insert(dev_id); | |||
} | |||
return true; | |||
} | |||
aclgrphProfConfig *aclgrphProfCreateConfig(uint32_t *deviceid_list, uint32_t device_nums, | |||
ProfilingAicoreMetrics aicore_metrics, ProfAicoreEvents *aicore_events, | |||
uint64_t data_type_config) { | |||
if (!isProfConfigValid(deviceid_list, device_nums)) { | |||
return nullptr; | |||
} | |||
aclgrphProfConfig *config = new (std::nothrow) aclgrphProfConfig(); | |||
if (config == nullptr) { | |||
GELOGE(INTERNAL_ERROR, "new aclgrphProfConfig fail"); | |||
return nullptr; | |||
} | |||
config->config.devNums = device_nums; | |||
if (memcpy_s(config->config.devIdList, sizeof(config->config.devIdList), deviceid_list, | |||
device_nums * sizeof(uint32_t)) != EOK) { | |||
GELOGE(INTERNAL_ERROR, "copy devID failed. size = %u", device_nums); | |||
delete config; | |||
return nullptr; | |||
} | |||
config->config.aicoreMetrics = static_cast<ProfAicoreMetrics>(aicore_metrics); | |||
config->config.dataTypeConfig = data_type_config; | |||
GELOGI("Successfully create prof config."); | |||
return config; | |||
} | |||
Status aclgrphProfDestroyConfig(aclgrphProfConfig *profiler_config) { | |||
if (profiler_config == nullptr) { | |||
GELOGE(PARAM_INVALID, "destroy profilerConfig failed, profilerConfig must not be nullptr"); | |||
return PARAM_INVALID; | |||
} | |||
delete profiler_config; | |||
GELOGI("Successfully destroy prof config."); | |||
return SUCCESS; | |||
} | |||
Status aclgrphProfStart(aclgrphProfConfig *profiler_config) { | |||
if (profiler_config == nullptr) { | |||
GELOGE(PARAM_INVALID, "aclgrphProfConfig is invalid."); | |||
return FAILED; | |||
} | |||
std::shared_ptr<GELib> instance_ptr = ge::GELib::GetInstance(); | |||
if (instance_ptr == nullptr || !instance_ptr->InitFlag()) { | |||
GELOGE(GE_CLI_GE_NOT_INITIALIZED, "Ge client is not initialized."); | |||
return FAILED; | |||
} | |||
std::lock_guard<std::mutex> lock(g_prof_mutex_); | |||
// if command mode is set, just return | |||
if (ProfilingManager::Instance().ProfilingOn()) { | |||
GELOGW("Graph prof finalize failed, cause profiling command pattern is running."); | |||
return GE_PROF_MODE_CONFLICT; | |||
} | |||
if (!g_graph_prof_init_) { | |||
GELOGE(GE_PROF_NOT_INIT, "Graph not profiling initialize."); | |||
return GE_PROF_NOT_INIT; | |||
} | |||
Status ret = ProfStartProfiling(&profiler_config->config); | |||
if (ret != SUCCESS) { | |||
GELOGE(ret, "Start profiling failed, prof result = %d", ret); | |||
return FAILED; | |||
} | |||
std::vector<string> prof_params; | |||
if (!TransProfConfigToParam(profiler_config, prof_params)) { | |||
GELOGE(PARAM_INVALID, "Transfer profilerConfig to string vector failed"); | |||
return PARAM_INVALID; | |||
} | |||
GraphLoader graph_loader; | |||
Command command; | |||
command.cmd_params.clear(); | |||
command.cmd_type = kProfilingStart; | |||
command.cmd_params = prof_params; | |||
command.module_index = profiler_config->config.dataTypeConfig; | |||
GELOGI("Profiling will start, device nums:%s , deviceID:[%s], data type config: 0x%llx", prof_params[0].c_str(), | |||
prof_params[kDeviceListIndex].c_str(), command.module_index); | |||
ret = graph_loader.CommandHandle(command); | |||
if (ret != SUCCESS) { | |||
GELOGE(ret, "Handle profiling command failed"); | |||
return FAILED; | |||
} | |||
GELOGI("Successfully execute GraphProfStartProfiling."); | |||
return SUCCESS; | |||
} | |||
Status aclgrphProfStop(aclgrphProfConfig *profiler_config) { | |||
if (profiler_config == nullptr) { | |||
GELOGE(PARAM_INVALID, "aclgrphProfConfig is invalid."); | |||
return FAILED; | |||
} | |||
std::shared_ptr<GELib> instance_ptr = ge::GELib::GetInstance(); | |||
if (instance_ptr == nullptr || !instance_ptr->InitFlag()) { | |||
GELOGE(GE_CLI_GE_NOT_INITIALIZED, "Ge client is not initialized."); | |||
return FAILED; | |||
} | |||
std::lock_guard<std::mutex> lock(g_prof_mutex_); | |||
// if command mode is set, just return | |||
if (ProfilingManager::Instance().ProfilingOn()) { | |||
GELOGW("Graph prof finalize failed, cause profiling command pattern is running."); | |||
return GE_PROF_MODE_CONFLICT; | |||
} | |||
if (!g_graph_prof_init_) { | |||
GELOGE(GE_PROF_NOT_INIT, "Graph not profiling initialize."); | |||
return GE_PROF_NOT_INIT; | |||
} | |||
for (uint32_t i = 0; i < profiler_config->config.devNums; i++) { | |||
uint64_t data_type_config; | |||
Status status = ProfGetDataTypeConfig(profiler_config->config.devIdList[i], data_type_config); | |||
if (status != SUCCESS) { | |||
GELOGE(status, "Prof get data type config failed, prof result = %d", status); | |||
return status; | |||
} | |||
if (data_type_config != profiler_config->config.dataTypeConfig) { | |||
GELOGE(FAILED, "data type config verify failed"); | |||
return FAILED; | |||
} | |||
} | |||
std::vector<string> prof_params; | |||
if (!TransProfConfigToParam(profiler_config, prof_params)) { | |||
GELOGE(PARAM_INVALID, "Transfer profilerConfig to string vector failed"); | |||
return PARAM_INVALID; | |||
} | |||
GraphLoader graph_loader; | |||
Command command; | |||
command.cmd_params.clear(); | |||
command.cmd_type = kProfilingStop; | |||
command.cmd_params = prof_params; | |||
command.module_index = profiler_config->config.dataTypeConfig; | |||
GELOGI("Profiling will stop, device nums:%s , deviceID:[%s], data type config: 0x%llx", prof_params[0].c_str(), | |||
prof_params[kDeviceListIndex].c_str(), command.module_index); | |||
Status ret = graph_loader.CommandHandle(command); | |||
if (ret != SUCCESS) { | |||
GELOGE(ret, "Handle profiling command failed"); | |||
return FAILED; | |||
} | |||
ret = ProfStopProfiling(&profiler_config->config); | |||
if (ret != SUCCESS) { | |||
GELOGE(ret, "Stop profiling failed, prof result = %d", ret); | |||
return ret; | |||
} | |||
GELOGI("Successfully execute GraphProfStopProfiling."); | |||
return SUCCESS; | |||
} | |||
} // namespace ge |
@@ -4,7 +4,6 @@ LOCAL_PATH := $(call my-dir) | |||
COMMON_LOCAL_SRC_FILES := \ | |||
proto/ge_api.proto \ | |||
ge_api.cc \ | |||
ge_prof.cc \ | |||
COMMON_LOCAL_C_INCLUDES := \ | |||
@@ -69,9 +68,9 @@ LOCAL_SHARED_LIBRARIES := \ | |||
libgraph \ | |||
libregister \ | |||
libge_compiler \ | |||
libge_common \ | |||
libmsprof | |||
libge_common | |||
LOCAL_STATIC_LIBRARIES += libmsprofiler_fwk \ | |||
LOCAL_LDFLAGS := -lrt -ldl | |||
@@ -104,8 +103,10 @@ LOCAL_SHARED_LIBRARIES := \ | |||
libregister \ | |||
libruntime \ | |||
libge_compiler \ | |||
libge_common \ | |||
libmsprof | |||
libge_common | |||
LOCAL_STATIC_LIBRARIES += libmsprofiler_fwk \ | |||
LOCAL_LDFLAGS := -lrt -ldl | |||
@@ -0,0 +1,199 @@ | |||
/** | |||
* Copyright 2020 Huawei Technologies Co., Ltd | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
*/ | |||
#include "common/profiling/ge_profiling.h" | |||
#include "runtime/base.h" | |||
#include "common/profiling/profiling_manager.h" | |||
#include "framework/common/debug/ge_log.h" | |||
#include "framework/common/debug/log.h" | |||
#include "graph/load/graph_loader.h" | |||
#include "init/gelib.h" | |||
#include "framework/common/ge_inner_error_codes.h" | |||
namespace { | |||
const uint32_t kDeviceListIndex = 3; | |||
const std::string kDeviceNums = "devNums"; | |||
const std::string kDeviceIdList = "devIdList"; | |||
const std::string kProfilingInit = "prof_init"; | |||
const std::string kProfilingFinalize = "prof_finalize"; | |||
const std::string kProfilingStart = "prof_start"; | |||
const std::string kProfilingStop = "prof_stop"; | |||
const std::string kProfModelSubscribe = "prof_model_subscribe"; | |||
const std::string kProfModelUnsubscribe = "prof_model_cancel_subscribe"; | |||
const std::string kRtSetDeviceRegName = "profiling"; | |||
const std::map<ProfCommandHandleType, std::string> kProfCommandTypeMap = { | |||
{kProfCommandhandleInit, kProfilingInit}, | |||
{kProfCommandhandleStart, kProfilingStart}, | |||
{kProfCommandhandleStop, kProfilingStop}, | |||
{kProfCommandhandleFinalize, kProfilingFinalize}, | |||
{kProfCommandhandleModelSubscribe, kProfModelSubscribe}, | |||
{kProfCommandhandleModelUnsubscribe, kProfModelUnsubscribe}}; | |||
} // namespace | |||
bool TransProfConfigToParam(const ProfCommandHandleData &profCommand, vector<string> &prof_config_params) { | |||
prof_config_params.clear(); | |||
prof_config_params.emplace_back(kDeviceNums); | |||
prof_config_params.emplace_back(std::to_string(profCommand.devNums)); | |||
prof_config_params.emplace_back(kDeviceIdList); | |||
std::string devID = ""; | |||
if (profCommand.devNums == 0) { | |||
GELOGW("The device num is invalid."); | |||
return false; | |||
} | |||
for (uint32_t i = 0; i < profCommand.devNums; i++) { | |||
devID.append(std::to_string(profCommand.devIdList[i])); | |||
if (i != profCommand.devNums - 1) { | |||
devID.append(","); | |||
} | |||
} | |||
prof_config_params.push_back(devID); | |||
return true; | |||
} | |||
bool isProfConfigValid(const uint32_t *deviceid_list, uint32_t device_nums) { | |||
if (deviceid_list == nullptr) { | |||
GELOGE(ge::PARAM_INVALID, "deviceIdList is nullptr"); | |||
return false; | |||
} | |||
if (device_nums == 0 || device_nums > MAX_DEV_NUM) { | |||
GELOGE(ge::PARAM_INVALID, "The device nums: %u is invalid.", device_nums); | |||
return false; | |||
} | |||
// real device num | |||
int32_t dev_count = 0; | |||
rtError_t rt_err = rtGetDeviceCount(&dev_count); | |||
if (rt_err != RT_ERROR_NONE) { | |||
GELOGE(ge::INTERNAL_ERROR, "Get the Device count fail."); | |||
return false; | |||
} | |||
if (device_nums > static_cast<uint32_t>(dev_count)) { | |||
GELOGE(ge::PARAM_INVALID, "Device num(%u) is not in range 1 ~ %d.", device_nums, dev_count); | |||
return false; | |||
} | |||
std::unordered_set<uint32_t> record; | |||
for (size_t i = 0; i < device_nums; ++i) { | |||
uint32_t dev_id = deviceid_list[i]; | |||
if (dev_id >= static_cast<uint32_t>(dev_count)) { | |||
GELOGE(ge::PARAM_INVALID, "Device id %u is not in range 0 ~ %d(exclude %d)", dev_id, dev_count, dev_count); | |||
return false; | |||
} | |||
if (record.count(dev_id) > 0) { | |||
GELOGE(ge::PARAM_INVALID, "Device id %u is duplicatedly set", dev_id); | |||
return false; | |||
} | |||
record.insert(dev_id); | |||
} | |||
return true; | |||
} | |||
ge::Status RegProfCtrlCallback(MsprofCtrlCallback func) { | |||
if (func == nullptr) { | |||
GELOGE(ge::PARAM_INVALID, "Msprof ctrl callback is nullptr."); | |||
return ge::PARAM_INVALID; | |||
} | |||
if (ge::ProfilingManager::Instance().GetMsprofCallback().msprofCtrlCallback != nullptr) { | |||
GELOGW("Msprof ctrl callback is exist, just ignore it."); | |||
} else { | |||
GELOGI("GE register Msprof ctrl callback."); | |||
ge::ProfilingManager::Instance().SetMsprofCtrlCallback(func); | |||
} | |||
return ge::SUCCESS; | |||
} | |||
ge::Status RegProfSetDeviceCallback(MsprofSetDeviceCallback func) { | |||
if (func == nullptr) { | |||
GELOGE(ge::PARAM_INVALID, "MsprofSetDeviceCallback callback is nullptr."); | |||
return ge::PARAM_INVALID; | |||
} | |||
// Pass MsprofSetDeviceCallback to runtime | |||
GELOGI("GE pass setdevice callback to runtime."); | |||
ge::Status rt_ret = rtRegDeviceStateCallback(kRtSetDeviceRegName.c_str(), static_cast<rtDeviceStateCallback>(func)); | |||
if (rt_ret != ge::SUCCESS) { | |||
GELOGE(rt_ret, "Pass MsprofSetDeviceCallback to runtime failed!"); | |||
return rt_ret; | |||
} | |||
return ge::SUCCESS; | |||
} | |||
ge::Status RegProfReporterCallback(MsprofReporterCallback func) { | |||
if (func == nullptr) { | |||
GELOGE(ge::PARAM_INVALID, "MsprofReporterCallback callback is nullptr."); | |||
return ge::PARAM_INVALID; | |||
} | |||
if (ge::ProfilingManager::Instance().GetMsprofCallback().msprofReporterCallback != nullptr) { | |||
GELOGW("Msprof reporter callback is exist, just ignore it."); | |||
} else { | |||
GELOGI("GE register Msprof reporter callback."); | |||
ge::ProfilingManager::Instance().SetMsprofReporterCallback(func); | |||
// Pass MsprofReporterCallback to runtime | |||
ge::Status rt_ret = rtSetMsprofReporterCallback(func); | |||
if (rt_ret != ge::SUCCESS) { | |||
GELOGE(rt_ret, "Pass MsprofReporterCallback to runtime failed!!"); | |||
return rt_ret; | |||
} | |||
// Pass MsprofReporterCallback to hccl | |||
} | |||
return ge::SUCCESS; | |||
} | |||
ge::Status ProfCommandHandle(ProfCommandHandleType type, void *data, uint32_t len) { | |||
if (type != kProfCommandhandleFinalize) { | |||
GE_CHECK_NOTNULL(data); | |||
} | |||
ProfCommandHandleData *prof_config_param = (ProfCommandHandleData *)data; | |||
auto iter = kProfCommandTypeMap.find(type); | |||
if (iter == kProfCommandTypeMap.end()) { | |||
GELOGW("The prof comand type is invalid."); | |||
return ge::PARAM_INVALID; | |||
} | |||
std::vector<string> prof_params; | |||
if (type == kProfCommandhandleStart || type == kProfCommandhandleStop) { | |||
if (!isProfConfigValid(prof_config_param->devIdList, prof_config_param->devNums)) { | |||
return ge::FAILED; | |||
} | |||
if (!TransProfConfigToParam(*prof_config_param, prof_params)) { | |||
GELOGE(ge::PARAM_INVALID, "Transfer profilerConfig to string vector failed"); | |||
return ge::PARAM_INVALID; | |||
} | |||
} | |||
ge::GraphLoader graph_loader; | |||
ge::Command command; | |||
command.cmd_params.clear(); | |||
command.cmd_type = iter->second; | |||
command.cmd_params = prof_params; | |||
if (type != kProfCommandhandleFinalize) { | |||
command.module_index = prof_config_param->profSwitch; | |||
} | |||
GELOGI("GE commandhandle execute, Command Type: %d, data type config: 0x%llx", type, command.module_index); | |||
if (type == kProfCommandhandleStart || type == kProfCommandhandleStop) { | |||
GELOGI("Profiling device nums:%s , deviceID:[%s]", prof_params[0].c_str(), prof_params[kDeviceListIndex].c_str()); | |||
} | |||
ge::Status ret = graph_loader.CommandHandle(command); | |||
if (ret != ge::SUCCESS) { | |||
GELOGE(ret, "Handle profiling command failed"); | |||
return ge::FAILED; | |||
} | |||
GELOGI("Successfully execute profiling command type: %d, command 0x%llx.", type, command.module_index); | |||
return ge::SUCCESS; | |||
} | |||
@@ -0,0 +1,26 @@ | |||
/** | |||
* Copyright 2020 Huawei Technologies Co., Ltd | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
*/ | |||
#include "common/profiling/ge_runner_profiling.h" | |||
#include "init/gelib.h" | |||
bool IsInitialize() { | |||
std::shared_ptr<ge::GELib> instance_ptr = ge::GELib::GetInstance(); | |||
if (instance_ptr == nullptr || instance_ptr->InitFlag() == false) { | |||
return false; | |||
} | |||
return true; | |||
} |
@@ -24,16 +24,9 @@ | |||
#include "graph/load/new_model_manager/davinci_model.h" | |||
namespace { | |||
const char *const kJobID = "jobID"; | |||
const char *const kDeviceID = "deviceID"; | |||
const char *const kStartCfg = "startCfg"; | |||
const char *const kFeatures = "features"; | |||
const char *const kConf = "conf"; | |||
const char *const kEvents = "events"; | |||
const char *const kAiCoreEvents = "ai_core_events"; | |||
const char *const kName = "name"; | |||
const char *const kTraceID = "traceId"; | |||
const char *const kProfDir = "resultPath"; | |||
const char *const kTrainingTrace = "training_trace"; | |||
const char *const kFpPoint = "fp_point"; | |||
const char *const kBpPoint = "bp_point"; | |||
const size_t kReportMaxLen = 2048; | |||
const int32_t kMaxDeviceNum = 256; | |||
const std::string kConfigNumsdev = "devNums"; | |||
@@ -45,7 +38,13 @@ const std::string kProfModelUnsubscribe = "prof_model_cancel_subscribe"; | |||
} // namespace | |||
namespace ge { | |||
ProfilingManager::ProfilingManager() : subscribe_count_(0) {} | |||
ProfilingManager::ProfilingManager() : is_load_profiling_(false), | |||
is_execute_profiling_(false), | |||
is_training_trace_(false), | |||
subscribe_count_(0) { | |||
prof_cb_.msprofCtrlCallback = nullptr; | |||
prof_cb_.msprofReporterCallback = nullptr; | |||
} | |||
ProfilingManager::~ProfilingManager() {} | |||
@@ -58,44 +57,29 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ge::Status ProfilingManager::In | |||
#ifdef DAVINCI_SUPPORT_PROFILING | |||
vector<int32_t>().swap(device_id_); | |||
subscribe_count_ = 0; | |||
job_id_ = options.job_id; | |||
GELOGI("ProfilingManager::Init job_id:%s", job_id_.c_str()); | |||
GELOGI("ProfilingManager::Init job_id:%s", options.job_id.c_str()); | |||
Status ret; | |||
if (!recv_profiling_config_.empty()) { | |||
GELOGI("Profiling json config from acl:%s", recv_profiling_config_.c_str()); | |||
ret = InitFromAclCfg(recv_profiling_config_); | |||
} else { | |||
ret = InitFromOptions(options); | |||
if (ret == SUCCESS && is_load_profiling_) { | |||
device_id_.push_back(options.device_id); | |||
} | |||
} | |||
struct MsprofGeOptions prof_conf = {{ 0 }}; | |||
Status ret = InitFromOptions(options, prof_conf); | |||
if (ret != SUCCESS) { | |||
GELOGE(ret, "Failed to init profiling."); | |||
return ret; | |||
} | |||
if (is_load_profiling_) { | |||
// register Framework to profiling | |||
int result = Msprof::Engine::Init(GE_PROFILING_MODULE, &engine_); | |||
if (result != 0) { | |||
GELOGE(FAILED, "Register profiling engine failed."); | |||
return FAILED; | |||
if (is_execute_profiling_) { | |||
if (prof_cb_.msprofCtrlCallback == nullptr) { | |||
GELOGE(ge::PARAM_INVALID, "MsprofCtrlCallback callback is nullptr."); | |||
return ge::PARAM_INVALID; | |||
} | |||
// profiling startup first time | |||
GELOGI("Begin to init profiling, device num %zu", device_id_.size()); | |||
for (size_t i = 0; i < device_id_.size(); ++i) { | |||
ret = StartProfiling(0, device_id_[i]); | |||
if (ret != SUCCESS) { | |||
GELOGW("Profiling start failed on device %d.", device_id_[i]); | |||
continue; | |||
} | |||
GELOGI("Profiling init succ on device %d.", device_id_[i]); | |||
int32_t cb_ret = prof_cb_.msprofCtrlCallback( | |||
static_cast<uint32_t>(MsprofCtrlCallbackType::MSPROF_CTRL_INIT_GE_OPTIONS), | |||
static_cast<void *>(&prof_conf), sizeof(MsprofGeOptions)); | |||
if (cb_ret != 0) { | |||
GELOGE(FAILED, "Call msprofCtrlCallback failed, type:%u, return:%d", | |||
static_cast<uint32_t>(MsprofCtrlCallbackType::MSPROF_CTRL_INIT_GE_OPTIONS), cb_ret); | |||
return FAILED; | |||
} | |||
GELOGI("Profiling init success"); | |||
} else { | |||
GELOGI("The profiling is off, skip the initialization"); | |||
} | |||
@@ -103,288 +87,116 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ge::Status ProfilingManager::In | |||
return SUCCESS; | |||
} | |||
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ge::Status ProfilingManager::InitFromAclCfg( | |||
const std::string &config) { | |||
ge::Status ProfilingManager::InitFromOptions(const Options &options, MsprofGeOptions &prof_conf) { | |||
#ifdef DAVINCI_SUPPORT_PROFILING | |||
try { | |||
is_load_profiling_ = false; | |||
is_execute_profiling_ = false; | |||
profiling_opts_.clear(); | |||
op_trace_conf_.clear(); | |||
Json start_prof_conf = Json::parse(config); | |||
Json &prof_conf = start_prof_conf[kStartCfg][0]; | |||
job_id_ = prof_conf[kJobID]; | |||
auto iter = prof_conf.find(kProfDir); | |||
if (iter != prof_conf.end()) { | |||
prof_dir_ = prof_conf[kProfDir]; | |||
} | |||
Json &device_id = prof_conf[kDeviceID]; | |||
if (device_id.size() != 0) { | |||
vector<int32_t>().swap(device_id_); | |||
bool is_all = false; | |||
for (size_t i = 0; i < device_id.size(); i++) { | |||
std::string device_id_str = device_id[i].get<std::string>(); | |||
if (device_id_str == "all") { | |||
is_all = true; | |||
break; | |||
} | |||
device_id_.push_back(std::stoi(device_id_str)); | |||
} | |||
if (is_all) { | |||
int32_t count = 0; | |||
rtError_t rt_err = rtGetDeviceCount(&count); | |||
if (rt_err != RT_ERROR_NONE) { | |||
GELOGE(FAILED, "Call rtGetDeviceCount to get device failed."); | |||
} | |||
vector<int32_t>().swap(device_id_); | |||
for (int32_t i = 0; i < count; ++i) { | |||
device_id_.push_back(i); | |||
} | |||
} | |||
// enable profiling by env | |||
char env_profiling_mode[MMPA_MAX_PATH] = { 0x00 }; | |||
is_load_profiling_ = false; // Change in ProfInit | |||
is_execute_profiling_ = false; | |||
if (options.profiling_mode == "1" && !options.profiling_options.empty()) { | |||
// enable profiling by ge option | |||
if (memcpy_s(prof_conf.options, MSPROF_OPTIONS_DEF_LEN_MAX, options.profiling_options.c_str(), | |||
options.profiling_options.size()) != EOK) { | |||
GELOGE(INTERNAL_ERROR, "copy profiling_options failed."); | |||
return INTERNAL_ERROR; | |||
} | |||
Json &features = prof_conf[kFeatures]; | |||
if (ParseFeaturesFromAclCfg(features) != SUCCESS) { | |||
GELOGE(FAILED, "Parse feature from acl cfg failed."); | |||
return FAILED; | |||
is_execute_profiling_ = true; | |||
GELOGI("The profiling in options is %s, %s. origin option: %s", options.profiling_mode.c_str(), | |||
prof_conf.options, options.profiling_options.c_str()); | |||
} else { | |||
(void)mmGetEnv("PROFILING_MODE", env_profiling_mode, MMPA_MAX_PATH); | |||
(void)mmGetEnv("PROFILING_OPTIONS", prof_conf.options, MSPROF_OPTIONS_DEF_LEN_MAX); | |||
// The env is invalid | |||
if ((strcmp("true", env_profiling_mode) != 0) || (strcmp(prof_conf.options, "\0") == 0)) { | |||
return SUCCESS; | |||
} | |||
is_load_profiling_ = true; | |||
// enable profiling by env | |||
is_execute_profiling_ = true; | |||
} catch (...) { | |||
GELOGE(FAILED, "Json conf is not invalid !"); | |||
GELOGI("The profiling in env is %s, %s", env_profiling_mode, prof_conf.options); | |||
} | |||
if (!is_execute_profiling_) { | |||
return SUCCESS; | |||
} | |||
// Parse json str for bp fp | |||
Status ret = ParseOptions(prof_conf.options); | |||
if (ret != ge::SUCCESS) { | |||
GELOGE(ge::PARAM_INVALID, "Parse training trace param failed."); | |||
return ge::PARAM_INVALID; | |||
} | |||
if (memcpy_s(prof_conf.jobId, sizeof(prof_conf.jobId), options.job_id.c_str(), | |||
sizeof(options.job_id.c_str())) != EOK) { | |||
GELOGE(INTERNAL_ERROR, "copy job_id failed."); | |||
return INTERNAL_ERROR; | |||
} | |||
#endif | |||
return ge::SUCCESS; | |||
} | |||
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ge::Status ProfilingManager::ParseFeaturesFromAclCfg( | |||
const Json &features) { | |||
#ifdef DAVINCI_SUPPORT_PROFILING | |||
ge::Status ProfilingManager::ParseOptions(const std::string &options) { | |||
if (options.empty()) { | |||
GELOGE(ge::PARAM_INVALID, "Profiling options is empty."); | |||
return ge::PARAM_INVALID; | |||
} | |||
try { | |||
for (size_t i = 0; i < features.size(); ++i) { | |||
const Json &feature = features[i]; | |||
if ((feature.find(kName) == feature.end()) || feature[kName].is_null()) { | |||
continue; | |||
} | |||
const std::string &name = feature[kName]; | |||
if (name == "op_trace") { | |||
const Json &conf = feature[kConf]; | |||
const Json &events = conf[0][kEvents]; | |||
const std::string &ai_core_events = events[0][kAiCoreEvents]; | |||
GELOGI("Op trace config from acl ai_core_events:%s", ai_core_events.c_str()); | |||
is_op_trace_ = true; | |||
ProfMgrConf prof_mgr_conf; | |||
int result = ProfMgrGetConf(ai_core_events, &prof_mgr_conf); | |||
if (result != 0) { | |||
GELOGE(FAILED, "ProfMgrGetConf failed."); | |||
return FAILED; | |||
} | |||
op_trace_conf_ = prof_mgr_conf.conf; | |||
op_trace_iter_num_ = static_cast<int32_t>(op_trace_conf_.size()); | |||
GELOGI("Op trace profiling iter num %d,", op_trace_iter_num_); | |||
} else if (name == "task_trace") { | |||
is_op_trace_ = false; | |||
if (feature.find(kConf) != feature.end()) { | |||
const Json &conf = feature[kConf]; | |||
std::stringstream task_trace_conf; | |||
task_trace_conf << conf; | |||
task_trace_conf_ = task_trace_conf.str(); | |||
} | |||
GELOGI("Task trace config from acl"); | |||
} else if (name == "system_trace") { | |||
is_op_trace_ = false; | |||
const Json &conf = feature[kConf]; | |||
std::stringstream system_trace_conf; | |||
system_trace_conf << conf; | |||
system_trace_conf_ = system_trace_conf.str(); | |||
GELOGI("System trace config from acl"); | |||
} | |||
profiling_opts_.push_back(name); | |||
Json prof_options = Json::parse(options); | |||
const std::string training_trace = prof_options[kTrainingTrace]; | |||
if (training_trace.empty()) { | |||
GELOGI("Training trace will not take effect."); | |||
return ge::SUCCESS; | |||
} | |||
GELOGI("GE profiling training trace:%s", training_trace.c_str()); | |||
if (training_trace != "on") { | |||
GELOGE(ge::PARAM_INVALID, "Training trace param:%s is invalid.", training_trace.c_str()); | |||
return ge::PARAM_INVALID; | |||
} | |||
fp_point_ = prof_options[kFpPoint]; | |||
bp_point_ = prof_options[kBpPoint]; | |||
if (!fp_point_.empty() && !bp_point_.empty()) { | |||
GELOGI("Training trace bp fp is set, bp_point:%s, fp_point:%s.", bp_point_.c_str(), fp_point_.c_str()); | |||
} | |||
} catch (...) { | |||
GELOGE(ge::PARAM_INVALID, "Json conf feature is not invalid !"); | |||
GELOGE(FAILED, "Json prof_conf options is invalid."); | |||
return ge::PARAM_INVALID; | |||
} | |||
#endif | |||
return ge::SUCCESS; | |||
} | |||
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ge::Status ProfilingManager::InitFromOptions(const Options &options) { | |||
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::StopProfiling() { | |||
#ifdef DAVINCI_SUPPORT_PROFILING | |||
// enable profiling support two ways: env and front end | |||
char profiling_mode_temp[MMPA_MAX_PATH] = { 0x00 }; | |||
char prof_options_temp[MMPA_MAX_PATH] = { 0x00 }; | |||
(void)mmGetEnv("PROFILING_MODE", profiling_mode_temp, MMPA_MAX_PATH); | |||
(void)mmGetEnv("PROFILING_OPTIONS", prof_options_temp, MMPA_MAX_PATH); | |||
const char *profiling_mode = profiling_mode_temp; | |||
const char *prof_options = prof_options_temp; | |||
if ((profiling_mode == nullptr) || (strcmp("true", profiling_mode) != 0) || (prof_options == nullptr)) { | |||
is_load_profiling_ = false; | |||
is_execute_profiling_ = false; | |||
} else { | |||
std::string prof_options_str = std::string(prof_options); | |||
profiling_opts_ = StringUtils::Split(prof_options_str, ':'); | |||
is_load_profiling_ = true; | |||
is_execute_profiling_ = true; | |||
GELOGI("The profiling in env is %s, %s", profiling_mode, prof_options); | |||
} | |||
if (!is_load_profiling_) { | |||
const std::string enable_profiling = "1"; | |||
if (options.profiling_mode != enable_profiling || options.profiling_options.empty()) { | |||
is_load_profiling_ = false; | |||
is_execute_profiling_ = false; | |||
return SUCCESS; | |||
} else { | |||
profiling_opts_ = StringUtils::Split(options.profiling_options, ':'); | |||
is_load_profiling_ = true; | |||
is_execute_profiling_ = true; | |||
GELOGI("The profiling in options is %s, %s", options.profiling_mode.c_str(), options.profiling_options.c_str()); | |||
} | |||
} | |||
// features:'training_trace', 'task_trace' or 'op_trace' etc | |||
if (!profiling_opts_.empty()) { | |||
if (profiling_opts_[0] == "op_trace") { | |||
is_op_trace_ = true; | |||
// op trace get conf | |||
ProfMgrConf prof_mgr_conf; | |||
int result = ProfMgrGetConf("", &prof_mgr_conf); | |||
if (result != 0) { | |||
GELOGE(FAILED, "ProfMgrGetConf failed."); | |||
return FAILED; | |||
} | |||
op_trace_conf_ = prof_mgr_conf.conf; | |||
op_trace_iter_num_ = static_cast<int32_t>(op_trace_conf_.size()); | |||
GELOGI("op trace profiling iter num %d,", op_trace_iter_num_); | |||
} else { | |||
is_op_trace_ = false; | |||
op_trace_iter_num_ = 1; | |||
uint64_t module = GetProfilingModule(); | |||
// The following if case will not be executed in normal case, inc case of ProfStopProfiling is abnormal | |||
int32_t device_num = static_cast<int32_t>(device_id_.size()); | |||
if (device_num != 0) { | |||
auto device_id_ptr = std::unique_ptr<uint32_t[]>(new (std::nothrow) uint32_t[device_num]); | |||
if (device_id_ptr == nullptr) { | |||
GELOGE(FAILED, "Stop profiling: device id ptr is null."); | |||
return; | |||
} | |||
} | |||
#endif | |||
return ge::SUCCESS; | |||
} | |||
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ge::Status ProfilingManager::StartProfiling(int32_t iter_num, | |||
int32_t device_id) { | |||
#ifdef DAVINCI_SUPPORT_PROFILING | |||
if (!profiling_opts_.empty()) { | |||
GELOGI("Start profiling index is %d", iter_num); | |||
// current one docker only use one device | |||
Json p_device; | |||
try { | |||
// profiling need physical_device_id | |||
p_device[kDeviceID] = std::to_string(device_id); | |||
p_device[kJobID] = job_id_; | |||
p_device[kTraceID] = std::to_string(GetContext().TraceId()); | |||
if (!prof_dir_.empty()) { | |||
p_device[kProfDir] = prof_dir_; | |||
GELOGI("Prof dir: %s.", prof_dir_.c_str()); | |||
} | |||
Json features; | |||
if (is_op_trace_) { | |||
Json f; | |||
f[kName] = "op_trace"; | |||
Json conf; | |||
if (op_trace_conf_.size() <= static_cast<size_t>(iter_num)) { | |||
GELOGE(FAILED, "Op trace iter num is invalid!"); | |||
return FAILED; | |||
} | |||
Json events; | |||
events[0] = nlohmann::json::parse(op_trace_conf_[iter_num]); | |||
conf[0][kEvents] = events; | |||
f[kConf] = conf; | |||
features[0] = f; | |||
if (iter_num == 0) { | |||
is_load_ = true; | |||
} | |||
} else { | |||
for (std::vector<std::string>::size_type i = 0; i < profiling_opts_.size(); i++) { | |||
Json f; | |||
if (profiling_opts_[i] == "system_trace") { | |||
f[kConf] = nlohmann::json::parse(system_trace_conf_); | |||
} else if (profiling_opts_[i] == "task_trace") { | |||
if (!task_trace_conf_.empty()) { | |||
f[kConf] = nlohmann::json::parse(task_trace_conf_); | |||
} | |||
} | |||
f[kName] = profiling_opts_[i]; | |||
features[i] = f; | |||
} | |||
is_load_ = true; | |||
} | |||
p_device[kFeatures] = features; | |||
// only one device, but sProfMgrStartUp API require for device list | |||
Json devices; | |||
devices[0] = p_device; | |||
Json start_cfg; | |||
start_cfg[kStartCfg] = devices; | |||
// convert json to string | |||
std::stringstream ss; | |||
ss << start_cfg; | |||
send_profiling_config_ = ss.str(); | |||
GELOGI("Profiling config %s\n", send_profiling_config_.c_str()); | |||
} catch (...) { | |||
GELOGE(FAILED, "Op trace json conf is not invalid !"); | |||
return FAILED; | |||
for (int32_t i = 0; i < device_num; i++) { | |||
device_id_ptr[i] = static_cast<uint32_t>(device_id_[i]); | |||
} | |||
// runtime startup for profiling | |||
uint64_t module = GetProfilingModule(); | |||
int32_t device_num = 1; | |||
uint32_t device_id_rt = static_cast<uint32_t>(device_id); | |||
GE_CHK_RT_RET(rtProfilerStart(module, device_num, &device_id_rt)); | |||
// call profiling startup API | |||
ProfMgrCfg prof_cfg = {send_profiling_config_}; | |||
void *prof_handle = ProfMgrStartUp(&prof_cfg); | |||
if (prof_handle == nullptr) { | |||
GELOGW("ProfMgrStartUp failed on device %d ", device_id); | |||
return FAILED; | |||
rtError_t rt_ret = rtProfilerStop(module, device_num, device_id_ptr.get()); | |||
if (rt_ret != RT_ERROR_NONE) { | |||
GELOGW("Call rtProfilerStop failed, ret:%d", rt_ret); | |||
} | |||
GELOGD("StartProfiling, prof_handle: %p", prof_handle); | |||
prof_handle_vec_.push_back(prof_handle); | |||
} | |||
#endif | |||
return SUCCESS; | |||
} | |||
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::StopProfiling() { | |||
#ifdef DAVINCI_SUPPORT_PROFILING | |||
Msprof::Engine::Reporter *reporter = PluginImpl::GetPluginReporter(); | |||
if (reporter != nullptr) { | |||
int ret = reporter->Flush(); | |||
GELOGI("Report data end, ret is %d", ret); | |||
// stop profiling | |||
if (prof_cb_.msprofCtrlCallback == nullptr) { | |||
GELOGE(ge::PARAM_INVALID, "MsprofCtrlCallback callback is nullptr."); | |||
return; | |||
} | |||
uint64_t module = GetProfilingModule(); | |||
int32_t device_num = static_cast<int32_t>(device_id_.size()); | |||
auto device_id_ptr = std::unique_ptr<uint32_t[]>(new (std::nothrow) uint32_t[device_num]); | |||
if (device_id_ptr == nullptr) { | |||
GELOGE(FAILED, "Stop profiling: device id ptr is null."); | |||
int32_t cb_ret = prof_cb_.msprofCtrlCallback(static_cast<uint32_t>(MsprofCtrlCallbackType::MSPROF_CTRL_FINALIZE), | |||
nullptr, 0); | |||
if (cb_ret != 0) { | |||
GELOGW("call msprofCtrlCallback failed, type:%u, return:%d", | |||
static_cast<uint32_t>(MsprofCtrlCallbackType::MSPROF_CTRL_FINALIZE), cb_ret); | |||
return; | |||
} | |||
for (int32_t i = 0; i < device_num; i++) { | |||
device_id_ptr[i] = static_cast<uint32_t>(device_id_[i]); | |||
} | |||
rtError_t rt_ret = rtProfilerStop(module, device_num, device_id_ptr.get()); | |||
if (rt_ret != RT_ERROR_NONE) { | |||
GELOGW("Call rtProfilerStop failed, ret:%d", rt_ret); | |||
} | |||
for (size_t i = 0; i < prof_handle_vec_.size(); ++i) { | |||
int result = ProfMgrStop(prof_handle_vec_[i]); | |||
if (result != 0) { | |||
GELOGW("ProfMgr stop return fail:%d, handle:%p", result, prof_handle_vec_[i]); | |||
} | |||
} | |||
vector<void *>().swap(prof_handle_vec_); | |||
is_load_ = false; | |||
recv_profiling_config_ = ""; | |||
GELOGI("Stop Profiling success."); | |||
#endif | |||
} | |||
@@ -392,12 +204,6 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::StopProf | |||
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::ProfilingTaskDescInfo( | |||
uint32_t model_id, const std::vector<TaskDescInfo> &task_desc_info, const int32_t &device_id) { | |||
#ifdef DAVINCI_SUPPORT_PROFILING | |||
Msprof::Engine::Reporter *reporter = PluginImpl::GetPluginReporter(); | |||
if (reporter == nullptr) { | |||
GELOGI("Profiling report is nullptr!"); | |||
return; | |||
} | |||
std::string data; | |||
for (const auto &task : task_desc_info) { | |||
std::string model_name = task.model_name; | |||
@@ -412,7 +218,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::Profilin | |||
.append(std::to_string(stream_id)).append(" ") | |||
.append(std::to_string(model_id)).append("\n")); | |||
Msprof::Engine::ReporterData reporter_data{}; | |||
ReporterData reporter_data{}; | |||
reporter_data.deviceId = device_id; | |||
reporter_data.data = (unsigned char *)data.c_str(); | |||
reporter_data.dataLen = data.size(); | |||
@@ -422,9 +228,9 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::Profilin | |||
return; | |||
} | |||
ret = reporter->Report(&reporter_data); | |||
if (ret != SUCCESS) { | |||
GELOGE(ret, "Reporter data of task_desc_info fail!"); | |||
int32_t cb_ret = CallMsprofReport(reporter_data); | |||
if (cb_ret != 0) { | |||
GELOGE(cb_ret, "Reporter data of task_desc_info failed, ret:%d", cb_ret); | |||
return; | |||
} | |||
} | |||
@@ -436,9 +242,6 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::Profilin | |||
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::ProfilingGraphDescInfo( | |||
uint32_t model_id, const std::vector<ComputeGraphDescInfo> &compute_graph_desc_info, const int32_t &device_id) { | |||
#ifdef DAVINCI_SUPPORT_PROFILING | |||
Msprof::Engine::Reporter *reporter = PluginImpl::GetPluginReporter(); | |||
GE_IF_BOOL_EXEC(reporter == nullptr, GELOGI("Profiling report is nullptr!"); return;); | |||
std::string data; | |||
for (const auto &graph : compute_graph_desc_info) { | |||
data.append("model_name:") | |||
@@ -493,64 +296,52 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::Profilin | |||
} | |||
data.append(" model_id:").append(std::to_string(model_id)); | |||
data.append("\n"); | |||
Msprof::Engine::ReporterData reporter_data{}; | |||
Report(device_id, data, *reporter, reporter_data); | |||
GraphDescReport(device_id, data); | |||
data.clear(); | |||
} | |||
#endif | |||
} | |||
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::Report( | |||
const int32_t &device_id, const string &data, Msprof::Engine::Reporter &reporter, | |||
Msprof::Engine::ReporterData &reporter_data) { | |||
void ProfilingManager::GraphDescReport(const int32_t &device_id, const string &data) { | |||
#ifdef DAVINCI_SUPPORT_PROFILING | |||
ReporterData reporter_data{}; | |||
int ret = -1; | |||
int32_t cb_ret = -1; | |||
size_t index = data.size() / kReportMaxLen; | |||
if (index >= 1) { | |||
reporter_data.deviceId = device_id; | |||
int ret = memcpy_s(reporter_data.tag, MSPROF_ENGINE_MAX_TAG_LEN + 1, "graph_desc_info", sizeof("graph_desc_info")); | |||
ret = memcpy_s(reporter_data.tag, MSPROF_ENGINE_MAX_TAG_LEN + 1, "graph_desc_info", sizeof("graph_desc_info")); | |||
GE_IF_BOOL_EXEC(ret != EOK, GELOGE(ret, "Report data tag of graph_desc_info memcpy error!"); return;); | |||
for (size_t i = 0; i < index; ++i) { | |||
reporter_data.data = (unsigned char *)data.c_str() + kReportMaxLen * i; | |||
reporter_data.dataLen = kReportMaxLen; | |||
ret = reporter.Report(&reporter_data); | |||
GE_IF_BOOL_EXEC(ret != SUCCESS, GELOGE(ret, "Reporter data of graph_desc_info fail!"); return;); | |||
cb_ret = CallMsprofReport(reporter_data); | |||
GE_IF_BOOL_EXEC(cb_ret != 0, GELOGE(cb_ret, "Reporter data of graph_desc_info failed, ret:%d", cb_ret); return;); | |||
} | |||
reporter_data.dataLen = data.size() - kReportMaxLen * index; | |||
if (reporter_data.dataLen != 0) { | |||
reporter_data.data = (unsigned char *)data.c_str() + kReportMaxLen * index; | |||
ret = reporter.Report(&reporter_data); | |||
GE_IF_BOOL_EXEC(ret != SUCCESS, GELOGE(ret, "Reporter data of graph_desc_info fail!"); return;); | |||
cb_ret = CallMsprofReport(reporter_data); | |||
GE_IF_BOOL_EXEC(cb_ret != 0, GELOGE(cb_ret, "Reporter data of graph_desc_info failed, ret:%d", cb_ret); return;); | |||
} | |||
} else { | |||
reporter_data.deviceId = device_id; | |||
reporter_data.data = (unsigned char *)data.c_str(); | |||
reporter_data.dataLen = data.size(); | |||
int ret = memcpy_s(reporter_data.tag, MSPROF_ENGINE_MAX_TAG_LEN + 1, "graph_desc_info", sizeof("graph_desc_info")); | |||
ret = memcpy_s(reporter_data.tag, MSPROF_ENGINE_MAX_TAG_LEN + 1, "graph_desc_info", sizeof("graph_desc_info")); | |||
GE_IF_BOOL_EXEC(ret != EOK, GELOGE(ret, "Report data tag of graph_desc_info memcpy error!"); return;); | |||
ret = reporter.Report(&reporter_data); | |||
GE_IF_BOOL_EXEC(ret != SUCCESS, GELOGE(ret, "Reporter data of graph_desc_info fail!"); return;); | |||
} | |||
#endif | |||
} | |||
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::PluginUnInit(const std::string &module) const { | |||
#ifdef DAVINCI_SUPPORT_PROFILING | |||
int ret = Msprof::Engine::UnInit(module); | |||
if (ret != SUCCESS) { | |||
GELOGE(ret, "profiling plugin uninit failed, ret:%d", ret); | |||
cb_ret = CallMsprofReport(reporter_data); | |||
GE_IF_BOOL_EXEC(cb_ret != 0, GELOGE(cb_ret, "Reporter data of graph_desc_info failed, ret:%d", cb_ret); return;); | |||
} | |||
#endif | |||
} | |||
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::ReportProfilingData( | |||
uint32_t model_id, const std::vector<TaskDescInfo> &task_desc_info, | |||
const std::vector<ComputeGraphDescInfo> &compute_graph_desc_info, | |||
bool check_device) { | |||
const std::vector<ComputeGraphDescInfo> &compute_graph_desc_info) { | |||
#ifdef DAVINCI_SUPPORT_PROFILING | |||
int32_t logic_device_id = 0; | |||
rtError_t rt_ret = rtGetDevice(&logic_device_id); | |||
@@ -559,13 +350,6 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::ReportPr | |||
return; | |||
} | |||
GELOGD("current logic_device_id:%d", logic_device_id); | |||
if (check_device) { | |||
auto ret = std::find(device_id_.begin(), device_id_.end(), logic_device_id); | |||
if (ret == device_id_.end()) { | |||
GELOGE(FAILED, "get valid phy_device_id failed, profiling report failed."); | |||
return; | |||
} | |||
} | |||
GELOGD("start ProfilingTaskDescInfo."); | |||
ProfilingTaskDescInfo(model_id, task_desc_info, logic_device_id); | |||
GELOGD("start ProfilingGraphDescInfo."); | |||
@@ -574,11 +358,6 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::ReportPr | |||
#endif | |||
} | |||
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::SetProfilingConfig( | |||
const std::string &profiling_cfg) { | |||
recv_profiling_config_ = profiling_cfg; | |||
} | |||
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY uint64_t ProfilingManager::GetProfilingModule() { | |||
uint64_t module = PROF_MODEL_EXECUTE_MASK | | |||
PROF_RUNTIME_API_MASK | | |||
@@ -594,9 +373,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY uint64_t ProfilingManager::GetP | |||
return module; | |||
} | |||
void ProfilingManager::UpdateSubscribeDeviceModuleMap(std::string prof_type, | |||
uint32_t device_id, | |||
uint64_t module) { | |||
void ProfilingManager::UpdateSubscribeDeviceModuleMap(std::string prof_type, uint32_t device_id, uint64_t module) { | |||
#ifdef DAVINCI_SUPPORT_PROFILING | |||
if (prof_type == kProfModelSubscribe) { | |||
if (subs_dev_module_.find(device_id) != subs_dev_module_.end()) { | |||
@@ -608,9 +385,13 @@ void ProfilingManager::UpdateSubscribeDeviceModuleMap(std::string prof_type, | |||
subs_dev_module_[device_id] = dev_info; | |||
} | |||
} else if (prof_type == kProfModelUnsubscribe) { | |||
if (subs_dev_module_.find(device_id) != subs_dev_module_.end()) { | |||
if (subs_dev_module_[device_id].subscribe_count > 0) { | |||
subs_dev_module_[device_id].subscribe_count--; | |||
auto iter = subs_dev_module_.find(device_id); | |||
if (iter != subs_dev_module_.end()) { | |||
if (iter->second.subscribe_count > 0) { | |||
iter->second.subscribe_count--; | |||
} | |||
if (iter->second.subscribe_count == 0) { | |||
subs_dev_module_.erase(iter); | |||
} | |||
} | |||
} else { | |||
@@ -626,10 +407,11 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfMo | |||
uint64_t model_load_mask = module & PROF_MODEL_LOAD_MASK; | |||
if ((subscribe_count_ == 0) && (model_load_mask == PROF_MODEL_LOAD_MASK)) { | |||
// register framework to profiling | |||
int32_t result = Msprof::Engine::Init(GE_PROFILING_MODULE, &engine_); | |||
if (result != SUCCESS) { | |||
GELOGE(FAILED, "Register profiling engine failed."); | |||
return FAILED; | |||
// register Framework to profiling | |||
int32_t cb_ret = PluginInit(); | |||
if (cb_ret != 0) { | |||
GELOGE(cb_ret, "profiling plugin init failed, ret:%d", cb_ret); | |||
return cb_ret; | |||
} | |||
GELOGI("Prof subscribe: model load profiling on."); | |||
} | |||
@@ -647,7 +429,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfMo | |||
UpdateSubscribeDeviceModuleMap(kProfModelSubscribe, device[0], module); | |||
// Report profiling data | |||
Status p_ret = davinci_model->ReportProfilingData(false); | |||
Status p_ret = davinci_model->ReportProfilingData(); | |||
if (p_ret != SUCCESS) { | |||
GELOGE(p_ret, "Report profiling data failed."); | |||
return p_ret; | |||
@@ -672,6 +454,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfMo | |||
auto iter = subs_dev_module_.find(device[0]); | |||
if (iter != subs_dev_module_.end()) { | |||
if (subs_dev_module_[device[0]].subscribe_count == 1) { | |||
// The same device_id, only stop at last time | |||
rtError_t rt_ret = rtProfilerStop(subs_dev_module_[device[0]].module, dev_num, device); | |||
if (rt_ret != RT_ERROR_NONE) { | |||
GELOGE(FAILED, "Runtime profiler stop failed."); | |||
@@ -679,15 +462,15 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfMo | |||
} | |||
} | |||
UpdateSubscribeDeviceModuleMap(kProfModelUnsubscribe, device[0], subs_dev_module_[device[0]].module); | |||
} else { | |||
GELOGE(FAILED, "The device_id:%u has not been subscribed, do not need to cancel.", device[0]); | |||
return FAILED; | |||
} | |||
subscribe_count_--; | |||
if (subscribe_count_ == 0) { | |||
int32_t ret = Msprof::Engine::UnInit(GE_PROFILING_MODULE); | |||
if (ret != SUCCESS) { | |||
GELOGE(ret, "Profiling plugin uninit failed, ret:%d", ret); | |||
return ret; | |||
} | |||
// profiling plugin uninit at last subscription | |||
PluginUnInit(); | |||
} | |||
#endif | |||
return SUCCESS; | |||
@@ -700,11 +483,12 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfIn | |||
if (model_load_mask == PROF_MODEL_LOAD_MASK) { | |||
// register Framework to profiling | |||
int32_t result = Msprof::Engine::Init(GE_PROFILING_MODULE, &engine_); | |||
if (result != SUCCESS) { | |||
GELOGE(FAILED, "Register profiling engine failed."); | |||
return FAILED; | |||
int32_t cb_ret = PluginInit(); | |||
if (cb_ret != 0) { | |||
GELOGE(cb_ret, "profiling plugin init failed, ret:%d", cb_ret); | |||
return cb_ret; | |||
} | |||
int32_t device_num = -1; | |||
rtError_t rt_ret = rtProfilerStart(model_load_mask, device_num, nullptr); | |||
if (rt_ret != RT_ERROR_NONE) { | |||
@@ -719,7 +503,6 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfIn | |||
if (training_trace_mask == PROF_TRAINING_TRACE_MASK) { | |||
is_training_trace_ = true; | |||
} | |||
is_acl_api_mode_ = true; | |||
GELOGI("Prof init success."); | |||
#endif | |||
return SUCCESS; | |||
@@ -730,19 +513,17 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfFi | |||
std::lock_guard<std::mutex> lock(mutex_); | |||
is_load_profiling_ = false; | |||
is_training_trace_ = false; | |||
is_acl_api_mode_ = false; | |||
is_execute_profiling_ = false; | |||
// profiling plugin uninit | |||
PluginUnInit(); | |||
int32_t ret = Msprof::Engine::UnInit(GE_PROFILING_MODULE); | |||
if (ret != SUCCESS) { | |||
GELOGE(ret, "Profiling plugin uninit failed, ret:%d", ret); | |||
} | |||
int32_t dev_num = -1; | |||
rtError_t rt_ret = rtProfilerStop(PROF_MODEL_LOAD_MASK, dev_num, nullptr); | |||
if (rt_ret != RT_ERROR_NONE) { | |||
GELOGE(FAILED, "Runtime profiler stop failed."); | |||
return FAILED; | |||
} | |||
for (auto device_id_module : device_id_module_map_) { | |||
if (device_id_module.second != 0) { | |||
uint32_t device_id = static_cast<uint32_t>(device_id_module.first); | |||
@@ -792,6 +573,7 @@ Status ProfilingManager::ProfParseDeviceId(const std::map<std::string, std::stri | |||
return FAILED; | |||
} catch (std::out_of_range &) { | |||
GELOGE(FAILED, "Device id: %s is out of range.", decvice_id[i].c_str()); | |||
return FAILED; | |||
} catch (...) { | |||
GELOGE(FAILED, "Device id: %s cannot change to int.", decvice_id[i].c_str()); | |||
return FAILED; | |||
@@ -818,6 +600,7 @@ Status ProfilingManager::ProfParseParam(const std::map<std::string, std::string> | |||
return FAILED; | |||
} catch (std::out_of_range &) { | |||
GELOGE(FAILED, "Device num: %s is out of range.", iter->second.c_str()); | |||
return FAILED; | |||
} catch (...) { | |||
GELOGE(FAILED, "Device num: %s cannot change to int.", iter->second.c_str()); | |||
return FAILED; | |||
@@ -859,7 +642,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfSt | |||
for (int32_t i = 0; i < device_num; i++) { | |||
device_id_ptr[i] = static_cast<uint32_t>(device_list[i]); | |||
} | |||
GELOGD("Runtime config param: 0x%llx, device num: %d.", module, device_num); | |||
GELOGI("Runtime config param: 0x%llx, device num: %d.", module, device_num); | |||
rtError_t rt_ret = rtProfilerStart(module, device_num, device_id_ptr.get()); | |||
if (rt_ret != RT_ERROR_NONE) { | |||
@@ -878,7 +661,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfSt | |||
GELOGW("Prof start: load model module is invalid."); | |||
} | |||
UpdateDeviceIdModuleMap(kProfStart, module, device_list); | |||
GELOGD("Prof start profiling success."); | |||
GELOGI("Prof start profiling success."); | |||
#endif | |||
return SUCCESS; | |||
} | |||
@@ -901,7 +684,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfSt | |||
for (int32_t i = 0; i < device_num; i++) { | |||
device_id_ptr[i] = static_cast<uint32_t>(device_list[i]); | |||
} | |||
GELOGD("Prof stop: runtime config param: 0x%llx, device num: %d", module, device_num); | |||
GELOGI("Prof stop: runtime config param: 0x%llx, device num: %d", module, device_num); | |||
rtError_t rt_ret = rtProfilerStop(module, device_num, device_id_ptr.get()); | |||
if (rt_ret != RT_ERROR_NONE) { | |||
GELOGE(FAILED, "Prof stop: runtime profiler config proc failed."); | |||
@@ -921,7 +704,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::ProfSt | |||
GELOGW("Prof stop: load model module is invalid."); | |||
} | |||
UpdateDeviceIdModuleMap(kProfStop, module, device_list); | |||
GELOGD("Prof stop profiling success."); | |||
GELOGI("Prof stop profiling success."); | |||
#endif | |||
return SUCCESS; | |||
} | |||
@@ -963,47 +746,90 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY bool ProfilingManager::Profilin | |||
if (rt_ret != RT_ERROR_NONE) { | |||
GELOGE(rt_ret, "Runtime get logic_device_id failed, current logic_device_id:%d", logic_device_id); | |||
} | |||
GELOGD("Current logic_device_id:%d", logic_device_id); | |||
GELOGI("Current logic_device_id:%d", logic_device_id); | |||
bool execute_model_prof_on = false; | |||
auto iter = std::find(device_id_.begin(), device_id_.end(), logic_device_id); | |||
if (iter != device_id_.end()) { | |||
execute_model_prof_on = true; | |||
} | |||
GELOGD("Flag is_execute_profiling: %d, execute_model_prof_on: %d", is_execute_profiling_, execute_model_prof_on); | |||
return is_execute_profiling_ || execute_model_prof_on; | |||
GELOGI("Flag is_execute_profiling: %d, execute_model_prof_on: %d", is_execute_profiling_, execute_model_prof_on); | |||
return execute_model_prof_on; | |||
} | |||
/** | |||
* @brief Profiling PluginImpl | |||
*/ | |||
// PluginImpl static variable init | |||
Msprof::Engine::Reporter *PluginImpl::reporter_ = nullptr; | |||
PluginImpl::PluginImpl(const std::string &module) : module_(module) { GELOGI("Create PluginImpl\n"); } | |||
int PluginImpl::Init(const Msprof::Engine::Reporter *reporter) { | |||
GELOGI("PluginImpl init"); | |||
reporter_ = const_cast<Msprof::Engine::Reporter *>(reporter); | |||
return 0; | |||
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::PluginInit() const { | |||
if (prof_cb_.msprofReporterCallback == nullptr) { | |||
GELOGE(ge::PARAM_INVALID, "MsprofReporterCallback callback is nullptr."); | |||
return ge::PARAM_INVALID; | |||
} | |||
return prof_cb_.msprofReporterCallback( | |||
static_cast<uint32_t>(MsprofReporterModuleId::MSPROF_MODULE_FRAMEWORK), | |||
static_cast<uint32_t>(MsprofReporterCallbackType::MSPROF_REPORTER_INIT), | |||
nullptr, 0); | |||
} | |||
int PluginImpl::UnInit() { | |||
GELOGI("PluginImpl Uninit"); | |||
reporter_ = nullptr; | |||
return 0; | |||
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::PluginUnInit() const { | |||
#ifdef DAVINCI_SUPPORT_PROFILING | |||
if (prof_cb_.msprofReporterCallback == nullptr) { | |||
GELOGE(ge::PARAM_INVALID, "MsprofReporterCallback callback is nullptr."); | |||
return; | |||
} | |||
int32_t cb_ret = prof_cb_.msprofReporterCallback( | |||
static_cast<uint32_t>(MsprofReporterModuleId::MSPROF_MODULE_FRAMEWORK), | |||
static_cast<uint32_t>(MsprofReporterCallbackType::MSPROF_REPORTER_UNINIT), | |||
nullptr, 0); | |||
if (cb_ret != 0) { | |||
GELOGW("profiling plugin uninit failed, ret:%d", cb_ret); | |||
} | |||
#endif | |||
} | |||
Msprof::Engine::PluginIntf *ProfilingEngineImpl::CreatePlugin() { | |||
GELOGI(" Create Plugin"); | |||
return new (std::nothrow) PluginImpl(GE_PROFILING_MODULE); | |||
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY Status ProfilingManager::CallMsprofReport( | |||
ReporterData &reporter_data) const { | |||
if (prof_cb_.msprofReporterCallback == nullptr) { | |||
GELOGE(ge::PARAM_INVALID, "MsprofReporterCallback callback is nullptr."); | |||
return ge::PARAM_INVALID; | |||
} | |||
return prof_cb_.msprofReporterCallback( | |||
static_cast<uint32_t>(MsprofReporterModuleId::MSPROF_MODULE_FRAMEWORK), | |||
static_cast<uint32_t>(MsprofReporterCallbackType::MSPROF_REPORTER_REPORT), | |||
static_cast<void *>(&reporter_data), sizeof(ReporterData)); | |||
} | |||
int ProfilingEngineImpl::ReleasePlugin(Msprof::Engine::PluginIntf *plugin) { | |||
if (plugin != nullptr) { | |||
delete plugin; | |||
plugin = nullptr; | |||
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void ProfilingManager::GetFpBpPoint( | |||
std::string &fp_point, std::string &bp_point) { | |||
// Env or options mode, fp_point_/bp_point_ have initiliazed on profiling init | |||
if (!fp_point_.empty() && !bp_point_.empty()) { | |||
fp_point = fp_point_; | |||
bp_point = bp_point_; | |||
GELOGI("Bp Fp have been initialized in env or options. bp_point: %s, fp_point: %s", bp_point.c_str(), fp_point.c_str()); | |||
return; | |||
} | |||
// ProfApi mode and training trace is set | |||
try { | |||
char env_profiling_options[MSPROF_OPTIONS_DEF_LEN_MAX] = { 0x00 }; | |||
INT32 ret = mmGetEnv("PROFILING_OPTIONS", env_profiling_options, MSPROF_OPTIONS_DEF_LEN_MAX); | |||
if (ret != EN_OK) { | |||
GELOGI("PROFILING_OPTIONS env is not exist."); | |||
return; | |||
} | |||
GELOGI("Parse env PROFILING_OPTIONS:%s.", env_profiling_options); | |||
Json prof_options = Json::parse(env_profiling_options); | |||
fp_point_ = prof_options[kFpPoint]; | |||
bp_point_ = prof_options[kBpPoint]; | |||
fp_point = fp_point_; | |||
bp_point = bp_point_; | |||
if (!fp_point_.empty() && !bp_point_.empty()) { | |||
GELOGI("Training trace bp fp is set, bp_point:%s, fp_point:%s.", bp_point_.c_str(), fp_point_.c_str()); | |||
} | |||
} catch (...) { | |||
GELOGE(FAILED, "Json prof options is invalid."); | |||
return; | |||
} | |||
return 0; | |||
return; | |||
} | |||
} // namespace ge |
@@ -26,9 +26,7 @@ | |||
#include "framework/common/ge_inner_error_codes.h" | |||
#include "framework/common/ge_types.h" | |||
#include "external/register/register_types.h" | |||
#include "toolchain/prof_engine.h" | |||
#include "toolchain/prof_mgr_core.h" | |||
#include "toolchain/prof_acl_api.h" | |||
#include "toolchain/prof_callback.h" | |||
using std::map; | |||
using std::string; | |||
@@ -37,35 +35,33 @@ using Json = nlohmann::json; | |||
namespace { | |||
const std::string GE_PROFILING_MODULE = "Framework"; | |||
// DataTypeConfig MASK | |||
#define PROF_ACL_API_MASK 0x0001 | |||
#define PROF_TASK_TIME_MASK 0x0002 | |||
#define PROF_AICORE_METRICS_MASK 0x0004 | |||
#define PROF_AICPU_TRACE_MASK 0x0008 | |||
#define PROF_MODEL_EXECUTE_MASK 0x0010 | |||
#define PROF_RUNTIME_API_MASK 0x0020 | |||
#define PROF_RUNTIME_TRACE_MASK 0x0040 | |||
#define PROF_SCHEDULE_TIMELINE_MASK 0x0080 | |||
#define PROF_SCHEDULE_TRACE_MASK 0x0100 | |||
#define PROF_AIVECTORCORE_METRICS_MASK 0x0200 | |||
#define PROF_SUBTASK_TIME_MASK 0x0400 | |||
#define PROF_TRAINING_TRACE_MASK 0x0800 | |||
#define PROF_HCCL_TRACE_MASK 0x1000 | |||
#define PROF_DATA_PROCESS_MASK 0x2000 | |||
#define PROF_MODEL_LOAD_MASK 0x8000000000000000 | |||
} // namespace | |||
namespace ge { | |||
struct DeviceSubsInfo { | |||
uint64_t module; | |||
uint32_t subscribe_count; | |||
}; | |||
// register Plugin | |||
class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY PluginImpl : public Msprof::Engine::PluginIntf { | |||
public: | |||
explicit PluginImpl(const std::string &module); | |||
~PluginImpl() {} | |||
int Init(const Msprof::Engine::Reporter *reporter); | |||
int UnInit(); | |||
static Msprof::Engine::Reporter *GetPluginReporter() { return reporter_; } | |||
private: | |||
static Msprof::Engine::Reporter *reporter_; | |||
std::string module_; | |||
}; | |||
// register Engine | |||
class ProfilingEngineImpl : public Msprof::Engine::EngineIntf { | |||
public: | |||
ProfilingEngineImpl() {} | |||
~ProfilingEngineImpl() {} | |||
Msprof::Engine::PluginIntf *CreatePlugin(); | |||
int ReleasePlugin(Msprof::Engine::PluginIntf *plugin); | |||
struct MsprofCallback { | |||
MsprofCtrlCallback msprofCtrlCallback; | |||
MsprofReporterCallback msprofReporterCallback; | |||
}; | |||
class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ProfilingManager { | |||
@@ -73,68 +69,54 @@ class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ProfilingManager { | |||
ProfilingManager(); | |||
virtual ~ProfilingManager(); | |||
static ProfilingManager &Instance(); | |||
ge::Status Init(const Options &options); | |||
ge::Status InitFromOptions(const Options &options); | |||
ge::Status InitFromAclCfg(const std::string &config); | |||
ge::Status StartProfiling(int32_t iter, int32_t device_id); | |||
void UpdateSubscribeDeviceModuleMap(std::string prof_type, uint32_t device_id, uint64_t module); | |||
ge::Status ProfModelSubscribe(uint64_t module, void *model); | |||
ge::Status ProfModelUnsubscribe(void *model); | |||
ge::Status ProfInit(uint64_t module); | |||
ge::Status ProfFinalize(); | |||
ge::Status ProfStartProfiling(uint64_t module, const std::map<std::string, std::string> &config_para); | |||
ge::Status ProfStopProfiling(uint64_t module, const std::map<std::string, std::string> &config_para); | |||
Status Init(const Options &options); | |||
Status ProfInit(uint64_t module); | |||
Status ProfFinalize(); | |||
Status ProfStartProfiling(uint64_t module, const std::map<std::string, std::string> &config_para); | |||
Status ProfStopProfiling(uint64_t module, const std::map<std::string, std::string> &config_para); | |||
Status ProfModelSubscribe(uint64_t module, void *model); | |||
Status ProfModelUnsubscribe(void *model); | |||
void StopProfiling(); | |||
bool ProfilingOpTraceOn() const { return is_op_trace_; } | |||
bool ProfilingLoadFlag() const { return is_load_; } | |||
bool ProfilingTrainingTraceOn() const { return is_training_trace_; } | |||
bool ProfilingModelLoadOn() const { return is_load_profiling_; } | |||
bool ProfilingModelExecuteOn() const; | |||
bool ProfilingOn() const { return is_load_profiling_ && is_execute_profiling_; } // only used by command pattern | |||
bool IsAclApiMode() const { return is_acl_api_mode_; } | |||
int32_t GetOpTraceIterNum() const { return op_trace_iter_num_; } | |||
bool ProfilingOn() const { return is_load_profiling_ && is_execute_profiling_; } // is_execute_profiling_ only used by ge option and env | |||
void ReportProfilingData(uint32_t model_id, const std::vector<TaskDescInfo> &task_desc_info, | |||
const std::vector<ComputeGraphDescInfo> &compute_graph_desc_info, | |||
bool check_device); | |||
void Report(const int32_t &device_id, const string &data, Msprof::Engine::Reporter &reporter, | |||
Msprof::Engine::ReporterData &reporter_data); | |||
const std::vector<ComputeGraphDescInfo> &compute_graph_desc_info); | |||
void ProfilingTaskDescInfo(uint32_t model_id, const std::vector<TaskDescInfo> &task_desc_info, | |||
const int32_t &device_id); | |||
void ProfilingGraphDescInfo(uint32_t model_id, const std::vector<ComputeGraphDescInfo> &compute_graph_desc_info, | |||
const int32_t &device_id); | |||
void SetProfilingConfig(const string &profiling_cfg); | |||
vector<int32_t> GetProfilingDeviceId() const { return device_id_; } | |||
void PluginUnInit(const std::string &module) const; | |||
Status PluginInit() const; | |||
void PluginUnInit() const; | |||
Status CallMsprofReport(ReporterData &reporter_data) const; | |||
struct MsprofCallback &GetMsprofCallback() { return prof_cb_; } | |||
void SetMsprofCtrlCallback(MsprofCtrlCallback func) { prof_cb_.msprofCtrlCallback = func; } | |||
void SetMsprofReporterCallback(MsprofReporterCallback func) { prof_cb_.msprofReporterCallback = func; } | |||
void GetFpBpPoint(std::string &fp_point, std::string &bp_point); | |||
private: | |||
ge::Status ParseFeaturesFromAclCfg(const Json &feature); | |||
ge::Status ProfParseParam(const std::map<std::string, std::string> &config_para, int32_t &device_num, | |||
vector<int32_t> &device_list); | |||
ge::Status ProfParseDeviceId(const std::map<std::string, std::string> &config_para, | |||
Status InitFromOptions(const Options &options, MsprofGeOptions &prof_conf); | |||
Status ParseOptions(const std::string &options); | |||
Status ProfParseParam(const std::map<std::string, std::string> &config_para, int32_t &device_num, | |||
vector<int32_t> &device_list); | |||
Status ProfParseDeviceId(const std::map<std::string, std::string> &config_para, | |||
vector<int32_t> &device_list); | |||
uint64_t GetProfilingModule(); | |||
void GraphDescReport(const int32_t &device_id, const string &data); | |||
void UpdateDeviceIdModuleMap(string prof_type, uint64_t module, const vector<int32_t> &device_list); | |||
bool is_load_profiling_ = false; | |||
bool is_execute_profiling_ = false; | |||
bool is_op_trace_ = false; | |||
bool is_load_ = false; | |||
bool is_training_trace_ = false; | |||
bool is_acl_api_mode_ = false; | |||
int32_t op_trace_iter_num_ = 0; | |||
string job_id_; | |||
string prof_dir_; | |||
void UpdateSubscribeDeviceModuleMap(std::string prof_type, uint32_t device_id, uint64_t module); | |||
bool is_load_profiling_; | |||
bool is_execute_profiling_; | |||
bool is_training_trace_; | |||
vector<int32_t> device_id_; | |||
vector<string> op_trace_conf_; | |||
vector<string> profiling_opts_; | |||
vector<void *> prof_handle_vec_; | |||
string recv_profiling_config_; | |||
string send_profiling_config_; | |||
string system_trace_conf_; | |||
string task_trace_conf_; | |||
const ProfilingEngineImpl engine_; | |||
map<int32_t, uint64_t> device_id_module_map_; // key: device_id, value: profiling on module | |||
map<uint32_t, DeviceSubsInfo> subs_dev_module_; // key: device_id, value: profiling on module | |||
uint32_t subscribe_count_; | |||
std::mutex mutex_; | |||
MsprofCallback prof_cb_; | |||
std::string fp_point_; | |||
std::string bp_point_; | |||
}; | |||
} // namespace ge | |||
#endif // GE_COMMON_PROFILING_PROFILING_MANAGER_H_ |
@@ -17,6 +17,7 @@ set(SRC_LIST | |||
"../common/dump/dump_properties.cc" | |||
"../common/dump/dump_manager.cc" | |||
"../common/dump/dump_op.cc" | |||
"../common/profiling/ge_profiling.cc" | |||
"../graph/load/graph_loader.cc" | |||
"../graph/execute/graph_execute.cc" | |||
"../omm/csa_interact.cc" | |||
@@ -244,7 +245,6 @@ target_link_libraries(ge_executor_shared PRIVATE | |||
mmpa | |||
graph | |||
register | |||
msprof | |||
error_manager | |||
ascend_hal_stub | |||
ascend_protobuf | |||
@@ -304,7 +304,7 @@ Status GeExecutor::Finalize() { | |||
// Stop profiling | |||
if (ProfilingManager::Instance().ProfilingOn()) { | |||
ProfilingManager::Instance().StopProfiling(); | |||
ProfilingManager::Instance().PluginUnInit(GE_PROFILING_MODULE); | |||
ProfilingManager::Instance().PluginUnInit(); | |||
} | |||
GELOGI("Uninit GeExecutor over."); | |||
@@ -8,6 +8,7 @@ local_ge_executor_src_files := \ | |||
../common/dump/dump_op.cc \ | |||
../common/ge/plugin_manager.cc \ | |||
../common/ge/op_tiling_manager.cc \ | |||
../common/profiling/ge_profiling.cc \ | |||
../graph/load/graph_loader.cc \ | |||
../graph/execute/graph_execute.cc \ | |||
../omm/csa_interact.cc \ | |||
@@ -177,7 +178,6 @@ local_ge_executor_shared_library := \ | |||
libmmpa \ | |||
libgraph \ | |||
libregister \ | |||
libmsprof \ | |||
liberror_manager \ | |||
local_ge_executor_ldflags := -lrt -ldl \ | |||
@@ -234,7 +234,6 @@ LOCAL_SHARED_LIBRARIES := \ | |||
libmmpa \ | |||
libgraph \ | |||
libregister \ | |||
libmsprof \ | |||
liberror_manager \ | |||
stub/libascend_hal \ | |||
@@ -272,7 +271,6 @@ LOCAL_SHARED_LIBRARIES := \ | |||
libruntime \ | |||
libslog \ | |||
libmmpa \ | |||
libmsprof \ | |||
LOCAL_LDFLAGS += $(local_ge_executor_ldflags) | |||
@@ -304,7 +302,6 @@ LOCAL_SHARED_LIBRARIES := \ | |||
libruntime \ | |||
libslog \ | |||
libmmpa \ | |||
libmsprof \ | |||
ifeq ($(device_os),android) | |||
LOCAL_LDFLAGS += -ldl | |||
@@ -29,6 +29,8 @@ LIBGE_LOCAL_SRC_FILES := \ | |||
common/dump/dump_manager.cc \ | |||
common/dump/dump_properties.cc \ | |||
common/dump/dump_op.cc \ | |||
common/profiling/ge_profiling.cc \ | |||
common/profiling/ge_runner_profiling.cc \ | |||
engine_manager/dnnengine_manager.cc \ | |||
ge_local_engine/engine/host_cpu_engine.cc \ | |||
generator/ge_generator.cc \ | |||
@@ -307,7 +309,6 @@ LIBGE_LOCAL_SRC_FILES := \ | |||
LIBCLIENT_LOCAL_SRC_FILES := \ | |||
proto/ge_api.proto \ | |||
client/ge_api.cc \ | |||
client/ge_prof.cc \ | |||
RUNNER_LOCAL_C_INCLUDES := \ | |||
$(LOCAL_PATH) ./ \ | |||
@@ -372,7 +373,7 @@ LOCAL_SRC_FILES += $(LIBCLIENT_LOCAL_SRC_FILES) | |||
LOCAL_STATIC_LIBRARIES := libge_memory \ | |||
libadump_server \ | |||
libmsprofiler \ | |||
libmsprofiler_fwk \ | |||
libmmpa \ | |||
LOCAL_SHARED_LIBRARIES := \ | |||
@@ -382,7 +383,6 @@ LOCAL_SHARED_LIBRARIES := \ | |||
libgraph \ | |||
libregister \ | |||
libge_common \ | |||
libmsprof \ | |||
liberror_manager \ | |||
LOCAL_LDFLAGS := -lrt -ldl | |||
@@ -409,7 +409,6 @@ endif | |||
LOCAL_C_INCLUDES := $(RUNNER_LOCAL_C_INCLUDES) | |||
LOCAL_SRC_FILES := ../../out/ge/lib64/stub/ge_api.cc \ | |||
../../out/ge/lib64/stub/ge_prof.cc \ | |||
../../out/ge/lib64/stub/ge_ir_build.cc \ | |||
LOCAL_SHARED_LIBRARIES := | |||
@@ -465,7 +464,6 @@ LOCAL_SHARED_LIBRARIES := \ | |||
libc_sec \ | |||
libslog \ | |||
libmmpa \ | |||
libmsprof \ | |||
LOCAL_LDFLAGS := -lrt -ldl | |||
@@ -498,7 +496,6 @@ LOCAL_SHARED_LIBRARIES := \ | |||
libc_sec \ | |||
libslog \ | |||
libmmpa \ | |||
libmsprof \ | |||
LOCAL_LDFLAGS := -lrt -ldl | |||
@@ -49,8 +49,6 @@ const char *const kIsLastNode = "is_last_node"; | |||
const char *const kIsInputVar = "INPUT_IS_VAR"; | |||
const char *const kIsOutputVar = "OUTPUT_IS_VAR"; | |||
const char *const kProfilingMode = "PROFILING_MODE"; | |||
const char *const kProfilingFpPoint = "FP_POINT"; | |||
const char *const kProfilingBpPoint = "BP_POINT"; | |||
const uint32_t kProfilingArStep = 2; | |||
const uint64_t kProfilingFpStartLogid = 1; | |||
const uint64_t kProfilingBpEndLogid = 2; | |||
@@ -810,35 +808,23 @@ Status TaskGenerator::GetFpBpIndex(const ComputeGraphPtr &graph, ProfilingPoint | |||
vector<uint32_t> &all_reduce_nodes, std::string &fp_point_str, | |||
std::string &bp_point_str) const { | |||
if (ge::GetContext().GetOption(OPTION_EXEC_PROFILING_FPPONIT_OPTIONS, fp_point_str) == SUCCESS && | |||
ge::GetContext().GetOption(OPTION_EXEC_PROFILING_BPPONIT_OPTIONS, bp_point_str) == SUCCESS && | |||
!fp_point_str.empty() && !bp_point_str.empty()) { | |||
return SUCCESS; | |||
} | |||
ProfilingManager::Instance().GetFpBpPoint(fp_point_str, bp_point_str); | |||
Status ret = SUCCESS; | |||
const char *fp_point = std::getenv(kProfilingFpPoint); | |||
if (fp_point == nullptr) { | |||
if (fp_point_str.empty()) { | |||
ret = AutoFindFpOpIndex(graph, profiling_point); | |||
if (ret != SUCCESS) { | |||
GELOGW("First forward profiling op_index not set and FindFpOpIndex failed."); | |||
return FAILED; | |||
} | |||
} else { | |||
fp_point_str = string(fp_point); | |||
GELOGI("Get fp_point_str from env %s", fp_point_str.c_str()); | |||
} | |||
const char *bp_point = std::getenv(kProfilingBpPoint); | |||
if (bp_point == nullptr) { | |||
if (bp_point_str.empty()) { | |||
ret = AutoFindBpOpIndex(graph, profiling_point, all_reduce_nodes); | |||
if (ret != SUCCESS) { | |||
GELOGW("Last backward profiling op_index not set and FindBpOpIndex failed."); | |||
return FAILED; | |||
} | |||
} else { | |||
bp_point_str = string(bp_point); | |||
GELOGI("Get bp_point_str from env %s", bp_point_str.c_str()); | |||
} | |||
return SUCCESS; | |||
@@ -712,7 +712,7 @@ Status DavinciModel::Init(void *dev_ptr, size_t mem_size, void *weight_ptr, size | |||
// collect profiling for ge | |||
auto &profiling_manager = ProfilingManager::Instance(); | |||
if (profiling_manager.ProfilingModelLoadOn()) { | |||
Status p_ret = ReportProfilingData(!profiling_manager.IsAclApiMode()); | |||
Status p_ret = ReportProfilingData(); | |||
if (p_ret != SUCCESS) { | |||
GELOGE(p_ret, "Report profiling data failed."); | |||
return p_ret; | |||
@@ -723,14 +723,14 @@ Status DavinciModel::Init(void *dev_ptr, size_t mem_size, void *weight_ptr, size | |||
return ret; | |||
} | |||
Status DavinciModel::ReportProfilingData(bool check_device) { | |||
Status DavinciModel::ReportProfilingData() { | |||
std::vector<ComputeGraphDescInfo> compute_graph_desc_info; | |||
Status ret = GetComputeGraphInfo(compute_graph_desc_info); | |||
if (ret != SUCCESS) { | |||
GELOGE(ret, "GetComputeGraphInfo failed."); | |||
return ret; | |||
} | |||
ProfilingManager::Instance().ReportProfilingData(model_id_, GetTaskDescInfo(), compute_graph_desc_info, check_device); | |||
ProfilingManager::Instance().ReportProfilingData(model_id_, GetTaskDescInfo(), compute_graph_desc_info); | |||
GE_CHK_STATUS(SinkModelProfile(), "Sink model profiler failed."); | |||
op_list_.clear(); | |||
@@ -2250,10 +2250,8 @@ inline int64_t SumSize(const vector<int64_t> &size_list) { | |||
Status DavinciModel::SinkModelProfile() { | |||
// profiling plugin must be registered | |||
Msprof::Engine::Reporter *reporter = PluginImpl::GetPluginReporter(); | |||
GE_IF_BOOL_EXEC(reporter == nullptr, GELOGI("Profiling report is nullptr!"); return SUCCESS); | |||
Msprof::Engine::ReporterData reporter_data{}; | |||
auto &prof_mgr = ProfilingManager::Instance(); | |||
ReporterData reporter_data{}; | |||
// report model data tag name | |||
std::string tag_name; | |||
tag_name.append("model_load_info_").append(std::to_string(this->Id())); | |||
@@ -2271,32 +2269,32 @@ Status DavinciModel::SinkModelProfile() { | |||
reporter_data.deviceId = device_id_; | |||
reporter_data.data = (unsigned char *)&name_len; | |||
reporter_data.dataLen = sizeof(int32_t); | |||
GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", | |||
this->Id()); | |||
GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, | |||
"Reporter data fail, model id:%u.", this->Id()); | |||
reporter_data.data = (unsigned char *)name.c_str(); | |||
reporter_data.dataLen = name.size(); | |||
GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", | |||
this->Id()); | |||
GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, | |||
"Reporter data fail, model id:%u.", this->Id()); | |||
uint32_t model_id = this->Id(); | |||
reporter_data.data = (unsigned char *)&model_id; | |||
reporter_data.dataLen = sizeof(uint32_t); | |||
GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", | |||
this->Id()); | |||
GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, | |||
"Reporter data fail, model id:%u.", this->Id()); | |||
// Load Start/End Time | |||
int64_t start_time = this->GetLoadBeginTime(); | |||
reporter_data.data = (unsigned char *)&start_time; | |||
reporter_data.dataLen = sizeof(int64_t); | |||
GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", | |||
this->Id()); | |||
GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, | |||
"Reporter data fail, model id:%u.", this->Id()); | |||
int64_t end_time = this->GetLoadEndTime(); | |||
reporter_data.data = (unsigned char *)&end_time; | |||
reporter_data.dataLen = sizeof(int64_t); | |||
GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", | |||
this->Id()); | |||
GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, | |||
"Reporter data fail, model id:%u.", this->Id()); | |||
int32_t task_num = task_list_.size(); | |||
std::multimap<uint32_t, uint32_t> op_id_map; | |||
@@ -2310,6 +2308,7 @@ Status DavinciModel::SinkModelProfile() { | |||
uint32_t op_num = fusion_op_info->original_op_names.size(); | |||
uint32_t task_id = task->GetTaskID(); | |||
if (op_num > 0) { | |||
GELOGI("task.id = %u, opNum = %u", task_id, op_num); | |||
op_id_map.insert(std::make_pair(fusion_op_info->op_index, task_id)); | |||
} | |||
} | |||
@@ -2352,39 +2351,39 @@ Status DavinciModel::SinkModelProfile() { | |||
int32_t fusion_op_name_len = fusion_op_name.size() == 0 ? 1 : fusion_op_name.size(); | |||
reporter_data.data = (unsigned char *)&fusion_op_name_len; | |||
reporter_data.dataLen = sizeof(int32_t); | |||
GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", | |||
this->Id()); | |||
GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, | |||
"Reporter data fail, model id:%u.", this->Id()); | |||
reporter_data.data = (unsigned char *)fusion_op_name.c_str(); | |||
reporter_data.dataLen = fusion_op_name_len; | |||
GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", | |||
this->Id()); | |||
GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, | |||
"Reporter data fail, model id:%u.", this->Id()); | |||
// original op name before fusion | |||
reporter_data.data = (unsigned char *)&op_num; | |||
reporter_data.dataLen = sizeof(int32_t); | |||
GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", | |||
this->Id()); | |||
GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, | |||
"Reporter data fail, model id:%u.", this->Id()); | |||
for (uint32_t k = 0; k < op_num; k++) { | |||
std::string op_name = fusion_op_info->original_op_names[k]; | |||
int32_t op_name_len = op_name.size() == 0 ? 1 : op_name.size(); | |||
reporter_data.data = (unsigned char *)&op_name_len; | |||
reporter_data.dataLen = sizeof(int32_t); | |||
GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", | |||
this->Id()); | |||
GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, | |||
"Reporter data fail, model id:%u.", this->Id()); | |||
reporter_data.data = (unsigned char *)op_name.c_str(); | |||
reporter_data.dataLen = op_name_len; | |||
GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", | |||
this->Id()); | |||
GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, | |||
"Reporter data fail, model id:%u.", this->Id()); | |||
} | |||
// stream id info | |||
uint32_t streamId = task->GetStreamId(); | |||
reporter_data.data = (unsigned char *)&streamId; | |||
reporter_data.dataLen = sizeof(int32_t); | |||
GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", | |||
this->Id()); | |||
GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, | |||
"Reporter data fail, model id:%u.", this->Id()); | |||
// memory info | |||
struct memoryInfo memory_info; | |||
@@ -2400,22 +2399,22 @@ Status DavinciModel::SinkModelProfile() { | |||
memory_info.weight_size + memory_info.input_size + memory_info.output_size + memory_info.workspace_size; | |||
reporter_data.data = (unsigned char *)&memory_info; | |||
reporter_data.dataLen = sizeof(struct memoryInfo); | |||
GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", | |||
this->Id()); | |||
GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, | |||
"Reporter data fail, model id:%u.", this->Id()); | |||
// task info | |||
reporter_data.data = (unsigned char *)&task_count; | |||
reporter_data.dataLen = sizeof(uint32_t); | |||
GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", | |||
this->Id()); | |||
GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, | |||
"Reporter data fail, model id:%u.", this->Id()); | |||
Range task_range = op_id_map.equal_range(op_id); | |||
for (CIT idx = task_range.first; idx != task_range.second; ++idx) { | |||
uint32_t task_id = idx->second; | |||
reporter_data.data = (unsigned char *)&task_id; | |||
reporter_data.dataLen = sizeof(uint32_t); | |||
GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", | |||
this->Id()); | |||
GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, | |||
"Reporter data fail, model id:%u.", this->Id()); | |||
} | |||
} | |||
} | |||
@@ -2424,10 +2423,8 @@ Status DavinciModel::SinkModelProfile() { | |||
Status DavinciModel::SinkTimeProfile(const InputData ¤t_data) { | |||
// profiling plugin must be registered | |||
Msprof::Engine::Reporter *reporter = PluginImpl::GetPluginReporter(); | |||
GE_IF_BOOL_EXEC(reporter == nullptr, GELOGI("Profiling report is nullptr!"); return SUCCESS); | |||
Msprof::Engine::ReporterData reporter_data{}; | |||
auto &prof_mgr = ProfilingManager::Instance(); | |||
ReporterData reporter_data{}; | |||
// report model data tag name | |||
std::string tag_name; | |||
tag_name.append("model_time_info_") | |||
@@ -2450,33 +2447,33 @@ Status DavinciModel::SinkTimeProfile(const InputData ¤t_data) { | |||
size_t name_len = name.size(); | |||
reporter_data.data = (unsigned char *)&name_len; | |||
reporter_data.dataLen = sizeof(int32_t); | |||
GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", | |||
this->Id()); | |||
GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, | |||
"Reporter data fail, model id:%u.", this->Id()); | |||
reporter_data.data = (unsigned char *)name.c_str(); | |||
reporter_data.dataLen = name.size(); | |||
GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, "Reporter data fail, model id:%u.", | |||
this->Id()); | |||
GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, | |||
"Reporter data fail, model id:%u.", this->Id()); | |||
// request id | |||
uint64_t request_id = current_data.request_id; | |||
reporter_data.data = (unsigned char *)&request_id; | |||
reporter_data.dataLen = sizeof(uint32_t); | |||
GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, | |||
GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, | |||
"Reporter data fail, model id:%u, data index:%u.", this->Id(), current_data.index); | |||
// thread id | |||
int32_t thread_id = GetDataInputTid(); | |||
reporter_data.data = (unsigned char *)&thread_id; | |||
reporter_data.dataLen = sizeof(int32_t); | |||
GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, | |||
GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, | |||
"Reporter data fail, model id:%u, data index:%u.", this->Id(), current_data.index); | |||
// time info | |||
time_info_.modelId = this->Id(); | |||
reporter_data.data = (unsigned char *)&time_info_; | |||
reporter_data.dataLen = sizeof(struct timeInfo); | |||
GE_CHK_BOOL_EXEC(reporter->Report(&reporter_data) == SUCCESS, return FAILED, | |||
GE_CHK_BOOL_EXEC(prof_mgr.CallMsprofReport(reporter_data) == 0, return FAILED, | |||
"Reporter data fail, model id:%u, data index:%u.", this->Id(), current_data.index); | |||
return SUCCESS; | |||
@@ -2809,71 +2806,37 @@ void *DavinciModel::Run(DavinciModel *model) { | |||
} | |||
GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), model->SetProfileTime(MODEL_PRE_PROC_END)); | |||
GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), model->SetProfileTime(MODEL_INFER_START)); | |||
if (ProfilingManager::Instance().ProfilingOpTraceOn()) { | |||
GELOGI("GetOpTraceIterNum:%d", ProfilingManager::Instance().GetOpTraceIterNum()); | |||
for (int32_t i = 0; i < ProfilingManager::Instance().GetOpTraceIterNum(); i++) { | |||
if (!ProfilingManager::Instance().ProfilingLoadFlag()) { | |||
vector<int32_t> prof_device_id_vec = ProfilingManager::Instance().GetProfilingDeviceId(); | |||
for (size_t j = 0; j < prof_device_id_vec.size(); ++j) { | |||
// just profiling, no need to check value | |||
(void)ProfilingManager::Instance().StartProfiling(i, prof_device_id_vec[j]); | |||
} | |||
} | |||
GELOGI("rtModelExecute start."); | |||
rt_ret = rtModelExecute(model->rt_model_handle_, model->rt_model_stream_, 0); | |||
GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, rslt_flg = false; | |||
(void)model->ReturnResult(current_data.index, false, false, data_wrapper->GetOutput()); | |||
continue); // [No need to check value] | |||
GELOGI("rtModelExecute end"); | |||
GELOGI("rtStreamSynchronize start."); | |||
rt_ret = rtStreamSynchronize(model->rt_model_stream_); | |||
if (rt_ret == kModelAbortNormal || rt_ret == kModelAbortNormalNew) { | |||
GELOGI("The model with multiple datasets aborts normally."); | |||
} else { | |||
GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, rslt_flg = false; | |||
(void)model->ReturnResult(current_data.index, false, seq_end_flag, data_wrapper->GetOutput()); | |||
continue); // [No need to check value] | |||
} | |||
GELOGI("rtStreamSynchronize end."); | |||
(void)ProfilingManager::Instance().StopProfiling(); // just profiling, no need to check value | |||
} | |||
GE_TIMESTAMP_START(rtModelExecute); | |||
GELOGI("rtModelExecute start."); | |||
rt_ret = rtModelExecute(model->rt_model_handle_, model->rt_model_stream_, 0); | |||
GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, rslt_flg = false; | |||
(void)model->ReturnResult(current_data.index, false, false, data_wrapper->GetOutput()); | |||
CsaInteract::GetInstance().WriteErrorCode(rt_ret, ERROR_MODULE_RUNTIME, JOBSUBSTATE_GRAPH_EXEC); | |||
continue); | |||
GELOGI("rtModelExecute end"); | |||
GE_IF_BOOL_EXEC(model->is_first_execute_, GE_TIMESTAMP_EVENT_END(rtModelExecute, "GraphExcute::rtModelExecute")); | |||
GE_TIMESTAMP_START(rtStreamSynchronize); | |||
GELOGI("rtStreamSynchronize start."); | |||
rt_ret = rtStreamSynchronize(model->rt_model_stream_); | |||
if (rt_ret == kEndOfSequence || rt_ret == kEndOfSequenceNew) { | |||
seq_end_flag = true; | |||
} | |||
if (rt_ret == kModelAbortNormal || rt_ret == kModelAbortNormalNew) { | |||
GELOGI("The model with multiple datasets aborts normally."); | |||
} else { | |||
GE_TIMESTAMP_START(rtModelExecute); | |||
GELOGI("rtModelExecute start."); | |||
rt_ret = rtModelExecute(model->rt_model_handle_, model->rt_model_stream_, 0); | |||
GE_IF_BOOL_EXEC(rt_ret != RT_ERROR_NONE, rslt_flg = false; | |||
(void)model->ReturnResult(current_data.index, false, false, data_wrapper->GetOutput()); | |||
CsaInteract::GetInstance().WriteErrorCode(rt_ret, ERROR_MODULE_RUNTIME, JOBSUBSTATE_GRAPH_EXEC); | |||
continue); | |||
GELOGI("rtModelExecute end"); | |||
GE_IF_BOOL_EXEC(model->is_first_execute_, GE_TIMESTAMP_EVENT_END(rtModelExecute, "GraphExcute::rtModelExecute")); | |||
GE_TIMESTAMP_START(rtStreamSynchronize); | |||
GELOGI("rtStreamSynchronize start."); | |||
rt_ret = rtStreamSynchronize(model->rt_model_stream_); | |||
if (rt_ret == kEndOfSequence || rt_ret == kEndOfSequenceNew) { | |||
seq_end_flag = true; | |||
} | |||
if (rt_ret == kModelAbortNormal || rt_ret == kModelAbortNormalNew) { | |||
GELOGI("The model with multiple datasets aborts normally."); | |||
} else { | |||
GE_IF_BOOL_EXEC( | |||
rt_ret != RT_ERROR_NONE, rslt_flg = false; GELOGI("seq_end_flg: %d", seq_end_flag); | |||
(void)model->ReturnResult(current_data.index, false, seq_end_flag, | |||
data_wrapper->GetOutput()); // [No need to check value] | |||
CsaInteract::GetInstance().StoreInternalErrorCode(rt_ret, ERROR_MODULE_RUNTIME, JOBSUBSTATE_GRAPH_EXEC); | |||
continue); | |||
} | |||
GELOGI("rtStreamSynchronize end."); | |||
GE_IF_BOOL_EXEC(model->is_first_execute_, | |||
GE_TIMESTAMP_EVENT_END(rtStreamSynchronize, "GraphExcute::Wait for rtStreamSynchronize")); | |||
GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), model->SetProfileTime(MODEL_INFER_END)); | |||
GE_IF_BOOL_EXEC( | |||
rt_ret != RT_ERROR_NONE, rslt_flg = false; GELOGI("seq_end_flg: %d", seq_end_flag); | |||
(void)model->ReturnResult(current_data.index, false, seq_end_flag, | |||
data_wrapper->GetOutput()); // [No need to check value] | |||
CsaInteract::GetInstance().StoreInternalErrorCode(rt_ret, ERROR_MODULE_RUNTIME, JOBSUBSTATE_GRAPH_EXEC); | |||
continue); | |||
} | |||
GELOGI("rtStreamSynchronize end."); | |||
GE_IF_BOOL_EXEC(model->is_first_execute_, | |||
GE_TIMESTAMP_EVENT_END(rtStreamSynchronize, "GraphExcute::Wait for rtStreamSynchronize")); | |||
GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), model->SetProfileTime(MODEL_INFER_END)); | |||
GE_IF_BOOL_EXEC(ProfilingManager::Instance().ProfilingModelExecuteOn(), | |||
model->SetProfileTime(MODEL_AFTER_PROC_START)); | |||
GE_TIMESTAMP_START(ReturnResult3); | |||
@@ -440,7 +440,7 @@ class DavinciModel { | |||
Status SinkTimeProfile(const InputData ¤t_data); | |||
Status ReportProfilingData(bool check_device = true); | |||
Status ReportProfilingData(); | |||
void SaveDumpOpInfo(const RuntimeParam &model_param, const OpDescPtr &op, uint32_t task_id, uint32_t stream_id) { | |||
data_dumper_.SaveDumpOpInfo(model_param, op, task_id, stream_id); | |||
@@ -40,9 +40,7 @@ const int kCmdParSize = 2; | |||
const int kDumpCmdPairSize = 2; | |||
const std::size_t kProfCmdParaMaxSize = 1000; | |||
const std::size_t kProfStartCmdParaSize = 2; | |||
const std::string kCmdTypeProfile = "profile"; | |||
const std::string kCmdTypeDump = "dump"; | |||
const std::string kCmdTypeProfiling = "profiling"; | |||
const std::string kCmdTypeProfInit = "prof_init"; | |||
const std::string kCmdTypeProfFinalize = "prof_finalize"; | |||
const std::string kCmdTypeProfStart = "prof_start"; | |||
@@ -632,8 +630,7 @@ Status ModelManager::Stop(uint32_t model_id) { | |||
/// | |||
Status ModelManager::HandleCommand(const Command &command) { | |||
static const std::map<std::string, std::function<uint32_t(const Command &)>> cmds = { | |||
{kCmdTypeProfile, HandleProfileCommand}, {kCmdTypeDump, HandleDumpCommand}, | |||
{kCmdTypeProfiling, HandleAclProfilingCommand}, {kCmdTypeProfInit, HandleProfInitCommand}, | |||
{kCmdTypeDump, HandleDumpCommand}, {kCmdTypeProfInit, HandleProfInitCommand}, | |||
{kCmdTypeProfFinalize, HandleProfFinalizeCommand}, {kCmdTypeProfStart, HandleProfStartCommand}, | |||
{kCmdTypeProfStop, HandleProfStopCommand}, | |||
{kCmdTypeProfModelSubscribe, HandleProfModelSubscribeCommand}, | |||
@@ -648,21 +645,6 @@ Status ModelManager::HandleCommand(const Command &command) { | |||
} | |||
} | |||
Status ModelManager::HandleAclProfilingCommand(const Command &command) { | |||
if (command.cmd_params.size() < kCmdParSize) { | |||
GELOGE(PARAM_INVALID, "When the cmd_type is 'profiling', the size of cmd_params must larger than 2."); | |||
return PARAM_INVALID; | |||
} | |||
std::string map_key = command.cmd_params[0]; | |||
std::string value = command.cmd_params[1]; | |||
if (map_key == PROFILE_CONFIG) { | |||
ProfilingManager::Instance().SetProfilingConfig(value); | |||
} | |||
return SUCCESS; | |||
} | |||
Status ModelManager::GetModelByCmd(const Command &command, | |||
std::shared_ptr<DavinciModel> &davinci_model) { | |||
if (command.cmd_params.size() < kCmdParSize) { | |||
@@ -809,29 +791,6 @@ Status ModelManager::HandleProfStopCommand(const Command &command) { | |||
return SUCCESS; | |||
} | |||
Status ModelManager::HandleProfileCommand(const Command &command) { | |||
if (command.cmd_params.size() < kCmdParSize) { | |||
GELOGE(PARAM_INVALID, "When the cmd_type is 'profile', the size of cmd_params must larger than 2."); | |||
return PARAM_INVALID; | |||
} | |||
std::string map_key = command.cmd_params[0]; | |||
std::string value = command.cmd_params[1]; | |||
GELOGI("Profiling mode, Command key:%s , value:%s ", map_key.c_str(), value.c_str()); | |||
auto iter = PROFILE_COMPONENT_MAP.find(map_key); | |||
if (iter != PROFILE_COMPONENT_MAP.end()) { | |||
std::string property_value = (value == "on") ? "1" : "0"; | |||
PropertiesManager::Instance().SetPropertyValue(iter->second, property_value); | |||
} | |||
if ((map_key == PROFILER_JOBCTX || map_key == PROFILER_TARGET_PATH || map_key == RTS_PROFILE_PATH)) { | |||
PropertiesManager::Instance().SetPropertyValue(map_key, value); | |||
} | |||
return SUCCESS; | |||
} | |||
static Status ParserPara(const Command &command, const string &dump_key, string &dump_value) { | |||
auto iter = std::find(command.cmd_params.begin(), command.cmd_params.end(), dump_key); | |||
if (iter != command.cmd_params.end()) { | |||
@@ -169,8 +169,6 @@ class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY ModelManager { | |||
/// @brief comment handle function | |||
/// | |||
ge::Status HandleCommand(const Command &command); | |||
static ge::Status HandleAclProfilingCommand(const Command &command); | |||
static ge::Status HandleProfileCommand(const Command &command); | |||
static ge::Status HandleDumpCommand(const Command &command); | |||
static ge::Status HandleProfModelSubscribeCommand(const Command &command); | |||
static ge::Status HandleProfModelUnsubscribeCommand(const Command &command); | |||
@@ -260,8 +260,7 @@ Status NodeDoneCallback::ProfilingReport() { | |||
} | |||
auto &profiling_manager = ProfilingManager::Instance(); | |||
profiling_manager.ReportProfilingData(model->GetModelId(), task_desc_info, compute_graph_info, | |||
!profiling_manager.IsAclApiMode()); | |||
profiling_manager.ReportProfilingData(model->GetModelId(), task_desc_info, compute_graph_info); | |||
return SUCCESS; | |||
} | |||
@@ -485,11 +485,9 @@ Status GELib::Finalize() { | |||
void GELib::ShutDownProfiling() { | |||
std::lock_guard<std::mutex> lock(status_mutex_); | |||
if (!ProfilingManager::Instance().ProfilingOpTraceOn() && ProfilingManager::Instance().ProfilingOn()) { | |||
ProfilingManager::Instance().StopProfiling(); | |||
} | |||
if (ProfilingManager::Instance().ProfilingOn()) { | |||
ProfilingManager::Instance().PluginUnInit(GE_PROFILING_MODULE); | |||
ProfilingManager::Instance().StopProfiling(); | |||
ProfilingManager::Instance().PluginUnInit(); | |||
} | |||
} | |||
@@ -167,4 +167,5 @@ Status OpsKernelBuilderManager::GenerateTask(const Node &node, | |||
GELOGD("Done invoking GenerateTask successfully"); | |||
return SUCCESS; | |||
} | |||
} // namespace ge | |||
} // namespace ge |
@@ -72,8 +72,7 @@ Status ProfilingTaskInfo(OpTask *op_task) { | |||
std::vector<ComputeGraphDescInfo> compute_graph_info; | |||
auto &profiling_manager = ProfilingManager::Instance(); | |||
profiling_manager.ReportProfilingData(model_id, task_desc_info, compute_graph_info, | |||
!profiling_manager.IsAclApiMode()); | |||
profiling_manager.ReportProfilingData(model_id, task_desc_info, compute_graph_info); | |||
return SUCCESS; | |||
} | |||
} // namespace | |||
@@ -71,7 +71,7 @@ max_code_len_per_line = 100 | |||
when DEBUG on | |||
""" | |||
white_list_for_debug = ["attr_value.h", "operator.h", "tensor.h", "graph.h", "operator_factory.h", | |||
"ge_ir_build.h", "ge_api.h", "ge_prof.h", "tensorflow_parser.h", "caffe_parser.h"] | |||
"ge_ir_build.h", "ge_api.h", "tensorflow_parser.h", "caffe_parser.h"] | |||
include_dir_key_words = ["ge", "graph", "parser"] | |||
DEBUG = True | |||
@@ -1,102 +0,0 @@ | |||
/** | |||
* Copyright 2019-2020 Huawei Technologies Co., Ltd | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
*/ | |||
#ifndef INC_EXTERNAL_GE_GE_PROF_H_ | |||
#define INC_EXTERNAL_GE_GE_PROF_H_ | |||
#include <map> | |||
#include <string> | |||
#include <vector> | |||
#include "ge/ge_api_error_codes.h" | |||
namespace ge { | |||
enum ProfDataTypeConfig { | |||
kProfTaskTime = 0x0002, | |||
kProfAiCoreMetrics = 0x0004, | |||
kProfAicpuTrace = 0x0008, | |||
kProfTrainingTrace = 0x0800, | |||
kProfHcclTrace = 0x1000 | |||
}; | |||
enum ProfilingAicoreMetrics { | |||
kAicoreArithmaticThroughput = 0, | |||
kAicorePipeline = 1, | |||
kAicoreSynchronization = 2, | |||
kAicoreMemory = 3, | |||
kAicoreInternalMemory = 4, | |||
kAicoreStall = 5 | |||
}; | |||
typedef struct ProfAicoreEvents ProfAicoreEvents; | |||
typedef struct aclgrphProfConfig aclgrphProfConfig; | |||
/// | |||
/// @ingroup AscendCL | |||
/// @brief Initialize the profiling and set profiling configuration path | |||
/// @param [in] profiler_path: configuration path of profiling | |||
/// @param [in] length: length of configuration path | |||
/// @return Status result of function | |||
/// | |||
Status aclgrphProfInit(const char *profiler_path, uint32_t length); | |||
/// | |||
/// @ingroup AscendCL | |||
/// @brief Finalize profiling | |||
/// @return Status result of function | |||
/// | |||
Status aclgrphProfFinalize(); | |||
/// | |||
/// @ingroup AscendCL | |||
/// @brief Create data of type aclgrphProfConfig | |||
/// @param [in] deviceid_list: device id list | |||
/// @param [in] device_nums: device numbers | |||
/// @param [in] aicore_metrics: type of aicore metrics | |||
/// @param [in] aicore_events: pointer to aicore events be reserved, only support NULL now | |||
/// @param [in] data_type_config: modules need profiling | |||
/// @return Status result of function | |||
/// | |||
aclgrphProfConfig *aclgrphProfCreateConfig(uint32_t *deviceid_list, uint32_t device_nums, | |||
ProfilingAicoreMetrics aicore_metrics, ProfAicoreEvents *aicore_events, | |||
uint64_t data_type_config); | |||
/// | |||
/// @ingroup AscendCL | |||
/// @brief Destroy data of type aclgrphProfConfig | |||
/// @param [in] profiler_config: config of profiling | |||
/// @return Status result of function | |||
/// | |||
Status aclgrphProfDestroyConfig(aclgrphProfConfig *profiler_config); | |||
/// | |||
/// @ingroup AscendCL | |||
/// @brief Start profiling of modules which is configured by profiler config | |||
/// @param [in] profiler_config: config of profiling | |||
/// @return Status result of function | |||
/// | |||
Status aclgrphProfStart(aclgrphProfConfig *profiler_config); | |||
/// | |||
/// @ingroup AscendCL | |||
/// @brief Stop profiling of modules which is configured by profiler config | |||
/// @param [in] profiler_config: config of profiling | |||
/// @return Status result of function | |||
/// | |||
Status aclgrphProfStop(aclgrphProfConfig *profiler_config); | |||
} // namespace ge | |||
#endif // INC_EXTERNAL_GE_GE_PROF_H_ |
@@ -0,0 +1,45 @@ | |||
/** | |||
* Copyright 2020 Huawei Technologies Co., Ltd | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
*/ | |||
#ifndef INC_FRAMEWORK_COMMON_GE_PROFILING_H_ | |||
#define INC_FRAMEWORK_COMMON_GE_PROFILING_H_ | |||
#include "ge/ge_api_error_codes.h" | |||
#include "toolchain/prof_callback.h" | |||
#define MAX_DEV_NUM (64) | |||
enum ProfCommandHandleType { | |||
kProfCommandhandleInit = 0, | |||
kProfCommandhandleStart, | |||
kProfCommandhandleStop, | |||
kProfCommandhandleFinalize, | |||
kProfCommandhandleModelSubscribe, | |||
kProfCommandhandleModelUnsubscribe | |||
}; | |||
struct ProfCommandHandleData { | |||
uint64_t profSwitch; | |||
uint32_t devNums; // length of device id list | |||
uint32_t devIdList[MAX_DEV_NUM]; | |||
uint32_t modelId; | |||
}; | |||
ge::Status RegProfCtrlCallback(MsprofCtrlCallback func); | |||
ge::Status RegProfSetDeviceCallback(MsprofSetDeviceCallback func); | |||
ge::Status RegProfReporterCallback(MsprofReporterCallback func); | |||
ge::Status ProfCommandHandle(ProfCommandHandleType type, void *data, uint32_t len); | |||
#endif // INC_FRAMEWORK_COMMON_GE_PROFILING_H_ |
@@ -0,0 +1,24 @@ | |||
/** | |||
* Copyright 2020 Huawei Technologies Co., Ltd | |||
* | |||
* Licensed under the Apache License, Version 2.0 (the "License"); | |||
* you may not use this file except in compliance with the License. | |||
* You may obtain a copy of the License at | |||
* | |||
* http://www.apache.org/licenses/LICENSE-2.0 | |||
* | |||
* Unless required by applicable law or agreed to in writing, software | |||
* distributed under the License is distributed on an "AS IS" BASIS, | |||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |||
* See the License for the specific language governing permissions and | |||
* limitations under the License. | |||
*/ | |||
#ifndef INC_FRAMEWORK_COMMON_GE_RUNNER_PROFILING_H_ | |||
#define INC_FRAMEWORK_COMMON_GE_RUNNER_PROFILING_H_ | |||
#include "profiling/ge_profiling.h" | |||
bool IsInitialize(); | |||
#endif // INC_FRAMEWORK_COMMON_GE_RUNNER_PROFILING_H_ |
@@ -1 +1 @@ | |||
Subproject commit c85822cd5404e40cb4ff2bfc9483062648c13c57 | |||
Subproject commit dba83744a3ffe3d5f89496e69bb65c50f800c299 |
@@ -1 +1 @@ | |||
Subproject commit 5bc8c38b37476e8f4b9391c96e4a2cca59e53d8e | |||
Subproject commit ce574894f13cd94749d1a3964a13e8c97c20434a |