From 3fb19326033a905d54dd2b41f94b97d65f7a9669 Mon Sep 17 00:00:00 2001 From: taoxudonghaha Date: Tue, 2 Feb 2021 15:01:17 +0800 Subject: [PATCH] reunit os verify --- CMakeLists.txt | 23 +++---- build.sh | 60 +++++++++---------- ge/ge_local_engine/CMakeLists.txt | 1 - ge/host_cpu_engine/CMakeLists.txt | 1 - metadef | 2 +- parser | 2 +- .../inc/register/op_kernel_registry.h | 5 +- .../fwkacllib/inc/register/op_registry.h | 3 +- 8 files changed, 43 insertions(+), 54 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2b80996f..b1981dcf 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -88,10 +88,8 @@ if (ENABLE_OPEN_SRC) find_module(static_mmpa libmmpa.a ${ASCEND_ATC_DIR}) find_module(error_manager liberror_manager.so ${ASCEND_ATC_DIR}) if(PLATFORM STREQUAL "train") - find_module(hccl libhccl.so ${ASCEND_RUNTIME_DIR}) find_module(adump_server libadump_server.a ${ASCEND_RUNTIME_DIR}) find_module(runtime libruntime.so ${ASCEND_RUNTIME_DIR}) - find_module(resource libresource.so ${ASCEND_RUNTIME_DIR}) find_module(error_manager liberror_manager.so ${ASCEND_RUNTIME_DIR}) find_module(msprofiler_fwk_ext libmsprofiler_fwk.a ${ASCEND_RUNTIME_DIR}) find_module(ascend_hal_stub libascend_hal.so ${ASCEND_DRIVER_DIR}/driver) @@ -101,12 +99,10 @@ if (ENABLE_OPEN_SRC) elseif(PLATFORM STREQUAL "inference") find_module(adump_server libadump_server.a ${ASCEND_ACL_DIR}) find_module(runtime libruntime.so ${ASCEND_ACL_DIR}) - find_module(runtime_compile libruntime_compile.so ${ASCEND_ATC_DIR}) - find_module(resource libresource.so ${ASCEND_ATC_DIR}) + find_module(runtime_compile libruntime_compile.so ${ASCEND_ATC_DIR}) find_module(error_manager liberror_manager.so ${ASCEND_ATC_DIR}) find_module(error_manager_static liberror_manager.a ${ASCEND_ACL_DIR}) find_module(msprofiler_ext libmsprofiler.a ${ASCEND_ACL_DIR}) - #find_module(ascendcl_static libascendcl.a ${ASCEND_ACL_DIR}) if(PRODUCT STREQUAL "flr3") elseif(PRODUCT STREQUAL "flr1") find_module(ascend_hal_stub libascend_hal.so ${ASCEND_DRIVER_DIR}/driver) @@ -116,17 +112,14 @@ if (ENABLE_OPEN_SRC) find_module(ascend_hal_stub libascend_hal.so ${ASCEND_DRIVER_DIR}) endif() elseif(PLATFORM STREQUAL "all") - find_module(msprofiler_ext libmsprofiler.a ${ASCEND_ACL_DIR}) - find_module(hccl libhccl.so ${ASCEND_RUNTIME_DIR}) - find_module(adump_server libadump_server.a ${ASCEND_ACL_DIR}) - find_module(runtime libruntime.so ${ASCEND_ACL_DIR}) - find_module(runtime_compile libruntime_compile.so ${ASCEND_ATC_DIR}) - find_module(resource libresource.so ${ASCEND_ATC_DIR}) - find_module(error_manager liberror_manager.so ${ASCEND_ATC_DIR}) - find_module(error_manager_static liberror_manager.a ${ASCEND_ACL_DIR}) + find_module(adump_server libadump_server.a ${ASCEND_RUNTIME_DIR}) + find_module(runtime libruntime.so ${ASCEND_RUNTIME_DIR}) + find_module(error_manager liberror_manager.so ${ASCEND_RUNTIME_DIR}) find_module(msprofiler_fwk_ext libmsprofiler_fwk.a ${ASCEND_RUNTIME_DIR}) - find_module(ascend_hal_stub libascend_hal.so ${ASCEND_DRIVER_DIR}/driver) - #find_module(ascendcl_static libascendcl.a ${ASCEND_ACL_DIR}) + find_module(ascend_hal_stub libascend_hal.so ${ASCEND_DRIVER_DIR}/driver) + find_module(runtime_compile libruntime_compile.so ${ASCEND_ATC_DIR}) + find_module(error_manager_static liberror_manager.a ${ASCEND_ACL_DIR}) + find_module(msprofiler_ext libmsprofiler.a ${ASCEND_ACL_DIR}) else() message(STATUS "PLATFORM param is invalid, should be train or inference, you choose nothing!") endif() diff --git a/build.sh b/build.sh index f2fafd48..7b6da560 100644 --- a/build.sh +++ b/build.sh @@ -166,14 +166,14 @@ build_graphengine() echo "execute command: cmake ${CMAKE_ARGS} .. failed." return 1 fi - COMMON_TARGET="ge_common engine fmk_parser parser_common _caffe_parser fmk_onnx_parser graph register engine_conf.json optimizer_priority.pbtxt " + COMMON_TARGET="ge_local_engine ge_local_opskernel_builder host_cpu_engine host_cpu_opskernel_builder ge_common engine fmk_parser parser_common _caffe_parser fmk_onnx_parser graph register engine_conf.json optimizer_priority.pbtxt " TARGET=${COMMON_TARGET} if [ "x${PLATFORM}" = "xtrain" ] then - TARGET="ge_runner ge_local_engine ge_local_opskernel_builder host_cpu_engine host_cpu_opskernel_builder fwk_atc.bin ${TARGET}" + TARGET="ge_runner fwk_atc.bin ${TARGET}" elif [ "x${PLATFORM}" = "xinference" ] then - TARGET="ge_compiler atc_ge_local_engine atc_ge_local_opskernel_builder atc_host_cpu_engine atc_host_cpu_opskernel_builder atc_atc.bin opensrc_ascendcl ${TARGET}" + TARGET="ge_compiler atc_atc.bin opensrc_ascendcl ${TARGET}" elif [ "X$ENABLE_GE_UT" = "Xon" ] then TARGET="ut_libgraph ut_libge_multiparts_utest ut_libge_others_utest ut_libge_kernel_utest ut_libge_distinct_load_utest" @@ -183,7 +183,7 @@ build_graphengine() elif [ "x${PLATFORM}" = "xall" ] then # build all the target - TARGET="" + TARGET="ge_runner ge_compiler fwk_atc.bin atc_atc.bin opensrc_ascendcl ${TARGET}" fi make ${VERBOSE} ${TARGET} -j${THREAD_NUM} && make install @@ -198,8 +198,6 @@ g++ -v mk_dir ${OUTPUT_PATH} build_graphengine || { echo "GraphEngine build failed."; return; } echo "---------------- GraphEngine build finished ----------------" -#cp -rf "${BUILD_PATH}/graphengine/"*.so "${OUTPUT_PATH}" -#rm -rf "${OUTPUT_PATH}/"libproto* rm -f ${OUTPUT_PATH}/libgmock*.so rm -f ${OUTPUT_PATH}/libgtest*.so rm -f ${OUTPUT_PATH}/lib*_stub.so @@ -209,10 +207,6 @@ find ${OUTPUT_PATH} -name "*.so*" -print0 | xargs -0 chmod 500 echo "---------------- GraphEngine output generated ----------------" -# if [[ "X$ENABLE_GE_ST" = "Xon" ]]; then -# cp ${BUILD_PATH}/graphengine/tests/st/st_resnet50_train ${OUTPUT_PATH} -# fi - if [[ "X$ENABLE_GE_UT" = "Xon" || "X$ENABLE_GE_COV" = "Xon" ]]; then cp ${BUILD_PATH}/tests/ut/common/graph/ut_libgraph ${OUTPUT_PATH} cp ${BUILD_PATH}/tests/ut/ge/ut_libge_multiparts_utest ${OUTPUT_PATH} @@ -220,9 +214,6 @@ if [[ "X$ENABLE_GE_UT" = "Xon" || "X$ENABLE_GE_COV" = "Xon" ]]; then cp ${BUILD_PATH}/tests/ut/ge/ut_libge_others_utest ${OUTPUT_PATH} cp ${BUILD_PATH}/tests/ut/ge/ut_libge_kernel_utest ${OUTPUT_PATH} -# if [[ "X${ENABLE_GE_UT_ONLY_COMPILE}" != "Xon" ]]; then -# export LD_LIBRARY_PATH=${D_LINK_PATH}/x86_64/:${BUILD_PATH}../third_party/prebuild/x86_64/:${BUILD_PATH}/graphengine/:/usr/local/HiAI/driver/lib64:/usr/local/HiAI/runtime/lib64:${LD_LIBRARY_PATH} -# echo ${LD_LIBRARY_PATH} ${OUTPUT_PATH}/ut_libgraph && ${OUTPUT_PATH}/ut_libge_multiparts_utest && ${OUTPUT_PATH}/ut_libge_distinct_load_utest && @@ -232,17 +223,14 @@ if [[ "X$ENABLE_GE_UT" = "Xon" || "X$ENABLE_GE_COV" = "Xon" ]]; then echo "!!! UT FAILED, PLEASE CHECK YOUR CHANGES !!!" exit 1; fi -# fi - -# if [[ "X$ENABLE_GE_COV" = "Xon" ]]; then - echo "Generating coverage statistics, please wait..." - cd ${BASEPATH} - rm -rf ${BASEPATH}/cov - mkdir ${BASEPATH}/cov - lcov -c -d build/tests/ut/ge -d build/tests/ut/common/graph/ -o cov/tmp.info - lcov -r cov/tmp.info '*/output/*' '*/build/opensrc/*' '*/build/proto/*' '*/third_party/*' '*/tests/*' '/usr/local/*' -o cov/coverage.info - cd ${BASEPATH}/cov - genhtml coverage.info + echo "Generating coverage statistics, please wait..." + cd ${BASEPATH} + rm -rf ${BASEPATH}/cov + mkdir ${BASEPATH}/cov + lcov -c -d build/tests/ut/ge -d build/tests/ut/common/graph/ -o cov/tmp.info + lcov -r cov/tmp.info '*/output/*' '*/build/opensrc/*' '*/build/proto/*' '*/third_party/*' '*/tests/*' '/usr/local/*' -o cov/coverage.info + cd ${BASEPATH}/cov + genhtml coverage.info fi # generate output package in tar form, including ut/st libraries/executables @@ -256,6 +244,8 @@ generate_package() ATC_PATH="atc/lib64" ATC_BIN_PATH="atc/bin" FWK_BIN_PATH="fwkacllib/bin" + FWK_INCLUDE_PATH="fwkacllib/include" + ATC_INCLUDE_PATH="atc/include" NNENGINE_PATH="plugin/nnengine/ge_config" OPSKERNEL_PATH="plugin/opskernel" @@ -277,6 +267,8 @@ generate_package() mk_dir "${OUTPUT_PATH}/${ACL_PATH}" mk_dir "${OUTPUT_PATH}/${ATC_BIN_PATH}" mk_dir "${OUTPUT_PATH}/${FWK_BIN_PATH}" + mk_dir "${OUTPUT_PATH}/${FWK_INCLUDE_PATH}" + mk_dir "${OUTPUT_PATH}/${ATC_INCLUDE_PATH}" cd "${OUTPUT_PATH}" @@ -289,10 +281,10 @@ generate_package() find ${OUTPUT_PATH}/${GRAPHENGINE_LIB_PATH} -maxdepth 1 -name libengine.so -exec cp -f {} ${OUTPUT_PATH}/${ATC_PATH}/${NNENGINE_PATH}/../ \; MAX_DEPTH=1 - if [ "x${PLATFORM}" = "xall" ] || [ "x${PLATFORM}" = "xinference" ] - then - MAX_DEPTH=2 - fi +# if [ "x${PLATFORM}" = "xall" ] || [ "x${PLATFORM}" = "xinference" ] +# then +# MAX_DEPTH=2 +# fi for lib in "${PLUGIN_OPSKERNEL[@]}"; do find ${OUTPUT_PATH}/${GRAPHENGINE_LIB_PATH} -maxdepth ${MAX_DEPTH} -name "$lib" -exec cp -f {} ${OUTPUT_PATH}/${FWK_PATH}/${OPSKERNEL_PATH} \; @@ -318,7 +310,15 @@ generate_package() find ./lib/atclib -name atc.bin -exec cp {} "${OUTPUT_PATH}/${ATC_BIN_PATH}" \; find ./lib/fwkacl -name atc.bin -exec cp {} "${OUTPUT_PATH}/${FWK_BIN_PATH}" \; find ${OUTPUT_PATH}/${GRAPHENGINE_LIB_PATH} -maxdepth 1 -name "libascendcl.so" -exec cp -f {} ${OUTPUT_PATH}/${ACL_PATH} \; - + + cp -r ${OUTPUT_PATH}/../metadef/inc/external/* ${ATC_INCLUDE_PATH} + cp -r ${OUTPUT_PATH}/../parser/inc/external/* ${ATC_INCLUDE_PATH} + cp -r ${OUTPUT_PATH}/../inc/external/* ${ATC_INCLUDE_PATH} + + cp -r ${OUTPUT_PATH}/../metadef/inc/external/* ${FWK_INCLUDE_PATH} + cp -r ${OUTPUT_PATH}/../parser/inc/external/* ${FWK_INCLUDE_PATH} + cp -r ${OUTPUT_PATH}/../inc/external/* ${FWK_INCLUDE_PATH} + if [ "x${PLATFORM}" = "xtrain" ] then tar -cf graphengine_lib.tar fwkacllib @@ -339,4 +339,4 @@ then find ./ -name graphengine_lib.tar -exec rm {} \; tar -cf graphengine_lib.tar lib fi -echo "---------------- GraphEngine package archive generated ----------------" \ No newline at end of file +echo "---------------- GraphEngine package archive generated ----------------" diff --git a/ge/ge_local_engine/CMakeLists.txt b/ge/ge_local_engine/CMakeLists.txt index cbc00d27..00142cfe 100755 --- a/ge/ge_local_engine/CMakeLists.txt +++ b/ge/ge_local_engine/CMakeLists.txt @@ -77,7 +77,6 @@ target_compile_options(atc_ge_local_engine PRIVATE ) target_compile_definitions(atc_ge_local_engine PRIVATE - COMPILE_OMG_PACKAGE google=ascend_private FUNC_VISIBILITY ) diff --git a/ge/host_cpu_engine/CMakeLists.txt b/ge/host_cpu_engine/CMakeLists.txt index 4b397a95..13cb7434 100644 --- a/ge/host_cpu_engine/CMakeLists.txt +++ b/ge/host_cpu_engine/CMakeLists.txt @@ -70,7 +70,6 @@ target_compile_options(atc_host_cpu_engine PRIVATE ) target_compile_definitions(atc_host_cpu_engine PRIVATE - COMPILE_OMG_PACKAGE google=ascend_private FUNC_VISIBILITY ) diff --git a/metadef b/metadef index 848cf412..25967258 160000 --- a/metadef +++ b/metadef @@ -1 +1 @@ -Subproject commit 848cf412caa9b42ce4e75ab4d0a147ec97dc579b +Subproject commit 2596725889c19c60a03440ab9e4e313070326ec0 diff --git a/parser b/parser index 756c64c5..6516132e 160000 --- a/parser +++ b/parser @@ -1 +1 @@ -Subproject commit 756c64c59e451a955e81b59d957ad55f96c27d89 +Subproject commit 6516132e2eaeea2bf51cc790d52c83709588f5d8 diff --git a/third_party/fwkacllib/inc/register/op_kernel_registry.h b/third_party/fwkacllib/inc/register/op_kernel_registry.h index 5fed8960..35fcc857 100644 --- a/third_party/fwkacllib/inc/register/op_kernel_registry.h +++ b/third_party/fwkacllib/inc/register/op_kernel_registry.h @@ -27,10 +27,7 @@ class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY OpKernelRegistry { using CreateFn = HostCpuOp* (*)(); ~OpKernelRegistry(); - static OpKernelRegistry& GetInstance() { - static OpKernelRegistry instance; - return instance; - } + static OpKernelRegistry& GetInstance(); bool IsRegistered(const std::string &op_type); diff --git a/third_party/fwkacllib/inc/register/op_registry.h b/third_party/fwkacllib/inc/register/op_registry.h index 318eb3ba..f7e37390 100644 --- a/third_party/fwkacllib/inc/register/op_registry.h +++ b/third_party/fwkacllib/inc/register/op_registry.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include "register/register.h" @@ -88,7 +89,7 @@ class FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY OpRegistry { std::unordered_map fusion_parse_params_by_op_fn_map_; std::unordered_map op_types_to_parse_subgraph_post_func_; std::unordered_map> remove_input_configure_map_; - std::unordered_map origin_type_to_om_type_; + std::map origin_type_to_om_type_; std::unordered_map parse_op_to_graph_fn_map_; std::unordered_map op_types_to_parse_subgraph_post_func_v2_; };