Browse Source

update cmake

tags/v1.1.0
taoxiangdong 4 years ago
parent
commit
9d6f983611
100 changed files with 2393 additions and 1908 deletions
  1. +36
    -114
      CMakeLists.txt
  2. +23
    -0
      cmake/FindModule.cmake
  3. +0
    -13
      cmake/external_libs/eigen.cmake
  4. +38
    -0
      cmake/external_libs/gflags.cmake
  5. +0
    -16
      cmake/external_libs/gtest.cmake
  6. +23
    -9
      cmake/external_libs/json.cmake
  7. +29
    -5
      cmake/external_libs/onnx.cmake
  8. +0
    -54
      cmake/external_libs/protobuf.cmake
  9. +59
    -0
      cmake/external_libs/protobuf_shared.cmake
  10. +43
    -0
      cmake/external_libs/protobuf_static.cmake
  11. +102
    -0
      cmake/external_libs/protoc.cmake
  12. +60
    -11
      cmake/external_libs/securec.cmake
  13. +0
    -371
      cmake/ge_utils.cmake
  14. +52
    -0
      cmake/intf_pub_android.cmake
  15. +32
    -0
      cmake/intf_pub_linux.cmake
  16. +24
    -0
      cmake/intf_pub_windows.cmake
  17. +829
    -374
      ge/CMakeLists.txt
  18. +0
    -0
      ge/README.md
  19. +58
    -60
      ge/analyzer/analyzer.cc
  20. +31
    -24
      ge/analyzer/analyzer.h
  21. +0
    -72
      ge/client/CMakeLists.txt
  22. +1
    -1
      ge/client/ge_api.cc
  23. +31
    -50
      ge/client/ge_prof.cc
  24. +1
    -0
      ge/client/proto/ge_api.proto
  25. +1
    -0
      ge/client/proto/ge_ir.proto
  26. +1
    -0
      ge/client/proto/insert_op.proto
  27. +396
    -0
      ge/client/proto/om.proto
  28. +1
    -0
      ge/client/proto/task.proto
  29. +162
    -100
      ge/common/CMakeLists.txt
  30. +33
    -14
      ge/common/auth/file_saver.cc
  31. +5
    -3
      ge/common/auth/file_saver.h
  32. +18
    -15
      ge/common/base64.h
  33. +1
    -1
      ge/common/context/ctx.cc
  34. +0
    -248
      ge/common/convert/pb2json.cc
  35. +0
    -68
      ge/common/convert/pb2json.h
  36. +4
    -2
      ge/common/cust_aicpu_kernel_store.cc
  37. +1
    -1
      ge/common/cust_aicpu_kernel_store.h
  38. +1
    -1
      ge/common/debug/memory_dumper.cc
  39. +1
    -1
      ge/common/debug/memory_dumper.h
  40. +1
    -1
      ge/common/dump/dump_manager.cc
  41. +1
    -1
      ge/common/dump/dump_manager.h
  42. +1
    -1
      ge/common/dump/dump_op.cc
  43. +1
    -1
      ge/common/dump/dump_op.h
  44. +8
    -8
      ge/common/dump/dump_properties.cc
  45. +6
    -6
      ge/common/dump/dump_properties.h
  46. +21
    -0
      ge/common/dump/dump_server.cc
  47. +1
    -1
      ge/common/fmk_error_codes.cc
  48. +4
    -4
      ge/common/formats/format_transfers/datatype_transfer.cc
  49. +1
    -1
      ge/common/formats/format_transfers/datatype_transfer.h
  50. +3
    -3
      ge/common/formats/format_transfers/format_transfer_c1hwncoc0_hwcn.cc
  51. +1
    -1
      ge/common/formats/format_transfers/format_transfer_c1hwncoc0_hwcn.h
  52. +1
    -2
      ge/common/formats/format_transfers/format_transfer_dhwcn_fracz3D.cc
  53. +1
    -2
      ge/common/formats/format_transfers/format_transfer_dhwcn_fracz3D.h
  54. +1
    -2
      ge/common/formats/format_transfers/format_transfer_dhwnc_fracz3D_transpose.cc
  55. +1
    -2
      ge/common/formats/format_transfers/format_transfer_dhwnc_fracz3D_transpose.h
  56. +9
    -13
      ge/common/formats/format_transfers/format_transfer_fractal_nz.cc
  57. +1
    -1
      ge/common/formats/format_transfers/format_transfer_fractal_nz.h
  58. +22
    -22
      ge/common/formats/format_transfers/format_transfer_fractal_z.cc
  59. +1
    -1
      ge/common/formats/format_transfers/format_transfer_fractal_z.h
  60. +9
    -9
      ge/common/formats/format_transfers/format_transfer_fractal_zz.cc
  61. +1
    -1
      ge/common/formats/format_transfers/format_transfer_fractal_zz.h
  62. +4
    -5
      ge/common/formats/format_transfers/format_transfer_fracz_hwcn.cc
  63. +1
    -1
      ge/common/formats/format_transfers/format_transfer_fracz_hwcn.h
  64. +4
    -5
      ge/common/formats/format_transfers/format_transfer_fracz_nchw.cc
  65. +1
    -1
      ge/common/formats/format_transfers/format_transfer_fracz_nchw.h
  66. +4
    -5
      ge/common/formats/format_transfers/format_transfer_fracz_nhwc.cc
  67. +1
    -1
      ge/common/formats/format_transfers/format_transfer_fracz_nhwc.h
  68. +4
    -4
      ge/common/formats/format_transfers/format_transfer_hwcn_c1hwncoc0.cc
  69. +1
    -1
      ge/common/formats/format_transfers/format_transfer_hwcn_c1hwncoc0.h
  70. +3
    -3
      ge/common/formats/format_transfers/format_transfer_nc1hwc0_nchw.cc
  71. +1
    -1
      ge/common/formats/format_transfers/format_transfer_nc1hwc0_nchw.h
  72. +3
    -3
      ge/common/formats/format_transfers/format_transfer_nc1hwc0_nhwc.cc
  73. +1
    -1
      ge/common/formats/format_transfers/format_transfer_nc1hwc0_nhwc.h
  74. +1
    -1
      ge/common/formats/format_transfers/format_transfer_nchw_fz_c04.cc
  75. +1
    -1
      ge/common/formats/format_transfers/format_transfer_nchw_fz_c04.h
  76. +8
    -8
      ge/common/formats/format_transfers/format_transfer_nchw_nc1hwc0.cc
  77. +1
    -1
      ge/common/formats/format_transfers/format_transfer_nchw_nc1hwc0.h
  78. +3
    -3
      ge/common/formats/format_transfers/format_transfer_nhwc_nc1hwc0.cc
  79. +1
    -1
      ge/common/formats/format_transfers/format_transfer_nhwc_nc1hwc0.h
  80. +19
    -19
      ge/common/formats/format_transfers/format_transfer_transpose.cc
  81. +2
    -1
      ge/common/formats/format_transfers/format_transfer_transpose.h
  82. +3
    -2
      ge/common/formats/formats.cc
  83. +3
    -3
      ge/common/formats/formats.h
  84. +53
    -8
      ge/common/formats/utils/formats_definitions.h
  85. +1
    -1
      ge/common/formats/utils/formats_trans_utils.cc
  86. +1
    -1
      ge/common/formats/utils/formats_trans_utils.h
  87. +1
    -1
      ge/common/fp16_t.cc
  88. +1
    -1
      ge/common/fp16_t.h
  89. +37
    -37
      ge/common/ge/datatype_util.cc
  90. +5
    -5
      ge/common/ge/datatype_util.h
  91. +1
    -1
      ge/common/ge/ge_util.h
  92. +3
    -3
      ge/common/ge/op_tiling_manager.cc
  93. +2
    -1
      ge/common/ge/op_tiling_manager.h
  94. +10
    -14
      ge/common/ge/plugin_manager.cc
  95. +1
    -1
      ge/common/ge/plugin_manager.h
  96. +5
    -41
      ge/common/ge/tbe_plugin_manager.cc
  97. +6
    -7
      ge/common/ge/tbe_plugin_manager.h
  98. +0
    -0
      ge/common/ge_common.mk
  99. +0
    -0
      ge/common/ge_format_util.cc
  100. +3
    -4
      ge/common/helper/model_cache_helper.cc

+ 36
- 114
CMakeLists.txt View File

@@ -1,133 +1,55 @@
# Copyright 2019-2020 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================

cmake_minimum_required(VERSION 3.14)
project (GraphEngine[CXX])
set(CMAKE_CXX_STANDARD 17)
add_compile_definitions(_GLIBCXX_USE_CXX11_ABI=0)

set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR})
set(GE_SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR})
set(GE_PROTO_DIR ${GE_SOURCE_DIR}/)
set(GE_CODE_DIR ${CMAKE_CURRENT_LIST_DIR})

if (NOT BUILD_PATH)
set(BUILD_PATH "${CMAKE_SOURCE_DIR}/build")
endif()
# architecture: aarch64 or x86_64
message(STATUS "System architecture: ${CMAKE_HOST_SYSTEM_PROCESSOR}")
# system: euleros or ubuntu
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
execute_process(
COMMAND bash "-c" "cat /etc/os-release | grep ^ID= | awk -F '=' '{print $2}'"
OUTPUT_VARIABLE SYSTEM_TYPE
)
MESSAGE(STATUS "System type: ${SYSTEM_TYPE}.")
endif()

# download json headers, rather than whole repository
include(${GE_SOURCE_DIR}/cmake/ge_utils.cmake)
include(${GE_SOURCE_DIR}/cmake/external_libs/json.cmake)
include(${GE_SOURCE_DIR}/cmake/external_libs/eigen.cmake)
include(${GE_SOURCE_DIR}/cmake/external_libs/gtest.cmake)
include(${GE_SOURCE_DIR}/cmake/external_libs/protobuf.cmake)
include(${GE_SOURCE_DIR}/cmake/external_libs/onnx.cmake)
include(${GE_SOURCE_DIR}/cmake/external_libs/securec.cmake)
set(CMAKE_SKIP_RPATH TRUE)
option(ENABLE_OPEN_SRC "Enable graphengine compile in opensource." FALSE)

if (ENABLE_OPEN_SRC)
set(HI_PYTHON python3.7)

include(cmake/external_libs/protobuf_shared.cmake)
include(cmake/external_libs/protoc.cmake)
include(cmake/external_libs/gflags.cmake)
include(cmake/external_libs/securec.cmake)
include(cmake/external_libs/json.cmake)
include(cmake/FindModule.cmake)
include(cmake/intf_pub_linux.cmake)

# for CPU/GPU mode, find c_sec and slog from local prebuild
if(NOT ENABLE_D AND NOT GE_ONLY)
set(GE_PREBUILD_PATH ${GE_SOURCE_DIR}/third_party/prebuild/${CMAKE_HOST_SYSTEM_PROCESSOR})
find_library(slog libslog.so ${GE_PREBUILD_PATH})
# if D_LINK_PATH is set in environment variables, search libraries in given path
elseif(DEFINED ENV{D_LINK_PATH})
# D_LINK_PATH is set
set(GE_LIB_PATH $ENV{D_LINK_PATH})
set(GE_SYS_ARCH "")
if(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "x86_64")
# x86 ubuntu
set(GE_SYS_ARCH "x86_64")
elseif(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "aarch64")
# arm euleros
set(GE_SYS_ARCH "aarch64")
else()
message(FATAL_ERROR "Running on a unsupported architecture: ${SYSTEM_TYPE}, build terminated")
endif()
set(GE_LIB_PATH ${GE_LIB_PATH}/${GE_SYS_ARCH})
find_library(slog libslog.so ${GE_LIB_PATH})
find_library(mmpa libmmpa.so ${GE_LIB_PATH})
find_library(runtime libruntime.so ${GE_LIB_PATH})
find_library(msprof libmsprof.so ${GE_LIB_PATH})
find_library(register libregister.so ${GE_LIB_PATH})
find_library(hccl libhccl.so ${GE_LIB_PATH})
find_library(resource libresource.so ${GE_LIB_PATH})
find_library(error_manager liberror_manager.so ${GE_LIB_PATH})
else()
# Ascend mode
if(DEFINED ENV{ASCEND_CUSTOM_PATH})
set(ASCEND_DIR $ENV{ASCEND_CUSTOM_PATH})
else()
set(ASCEND_DIR /usr/local/Ascend)
endif()

set(ASCEND_DRIVER_DIR ${ASCEND_DIR}/driver/lib64/common)
set(ASCEND_RUNTIME_DIR ${ASCEND_DIR}/fwkacllib/lib64)
find_library(slog libslog.so ${ASCEND_DRIVER_DIR})
find_library(mmpa libmmpa.so ${ASCEND_DRIVER_DIR})
find_library(msprof libmsprof.so ${ASCEND_DRIVER_DIR})

find_library(hccl libhccl.so ${ASCEND_RUNTIME_DIR})
find_library(runtime libruntime.so ${ASCEND_RUNTIME_DIR})
find_library(register libregister.so ${ASCEND_RUNTIME_DIR})
find_library(resource libresource.so ${ASCEND_RUNTIME_DIR})
find_library(error_manager liberror_manager.so ${ASCEND_RUNTIME_DIR})
endif()

# add compile flags
if ("${CMAKE_BUILD_TYPE}" STREQUAL "Debug")
message("Build in Debug mode")
set(CMAKE_C_FLAGS "-O0 -g -Wall -fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack -pipe -fPIC ${CMAKE_C_FLAGS}")
set(CMAKE_CXX_FLAGS "-O0 -g -Wall -fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack -pipe -fPIC ${CMAKE_CXX_FLAGS}")
if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -rdynamic")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -rdynamic")
endif()
find_module(slog libslog.so ${ASCEND_DRIVER_DIR})
find_module(mmpa libmmpa.so ${ASCEND_DRIVER_DIR})
find_module(msprof libmsprof.so ${ASCEND_DRIVER_DIR})
find_module(hccl libhccl.so ${ASCEND_RUNTIME_DIR})
find_module(adump_server libadump_server.a ${ASCEND_RUNTIME_DIR})
find_module(runtime libruntime.so ${ASCEND_RUNTIME_DIR})
find_module(runtime_compile libruntime_compile.so ${ASCEND_RUNTIME_DIR})
find_module(resource libresource.so ${ASCEND_RUNTIME_DIR})
find_module(error_manager liberror_manager.so ${ASCEND_RUNTIME_DIR})
find_module(ascend_hal_stub libascend_hal.so ${ASCEND_DRIVER_DIR})

set(METADEF_DIR ${CMAKE_CURRENT_LIST_DIR}/metadef)
set(PARSER_DIR ${CMAKE_CURRENT_LIST_DIR}/parser)
set(GE_DEPEND_DIR ${CMAKE_CURRENT_LIST_DIR}/..)

add_subdirectory(metadef)
#add_subdirectory(metadef/graph)
#add_subdirectory(metadef/register)
else()
set(CMAKE_C_FLAGS "-O2 -Wall -fPIC -fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack -pipe ${CMAKE_C_FLAGS}")
set(CMAKE_CXX_FLAGS "-O2 -Wall -fPIC -fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack -pipe ${CMAKE_CXX_FLAGS}")
endif ()

# force __FILE__ to show relative path of file, from source directory, as cmake project makes __FILE__ absolute directory
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D__FILE__='\"$(subst $(realpath ${CMAKE_SOURCE_DIR})/,,$(abspath $<))\"' -Wno-builtin-macro-redefined")

# compile libraries from following directories
# libgraph is compiled in any situation
add_subdirectory(${GE_SOURCE_DIR}/metadef/graph)
if(ENABLE_D)
# if MindSpore compiles in D mode, compile the following libraries
add_subdirectory(${GE_SOURCE_DIR}/ge/common)
add_subdirectory(${GE_SOURCE_DIR}/ge/ge_runtime)
elseif(GE_ONLY)
# standalone GraphEngine compiles all following libraries
add_subdirectory(${GE_SOURCE_DIR}/ge/common)
add_subdirectory(${GE_SOURCE_DIR}/ge/ge_runtime)
add_subdirectory(${GE_SOURCE_DIR}/ge/ge_local_engine)
add_subdirectory(${GE_SOURCE_DIR}/ge/graph/build/memory)
add_subdirectory(${GE_SOURCE_DIR}/ge/)
add_subdirectory(${GE_SOURCE_DIR}/ge/plugin/engine)
set(METADEF_DIR ${CMAKE_CURRENT_LIST_DIR}/../metadef)
set(PARSER_DIR ${CMAKE_CURRENT_LIST_DIR}/../parser)
set(GE_DEPEND_DIR ${CMAKE_CURRENT_LIST_DIR}/..)
endif()

# if (ENABLE_GE_COV OR ENABLE_GE_UT OR ENABLE_GE_ST)
# add_subdirectory(tests)
# endif()

add_subdirectory(ge)

+ 23
- 0
cmake/FindModule.cmake View File

@@ -0,0 +1,23 @@
#[[
module - the name of export imported target
name - find the library name
path - find the library path
#]]
function(find_module module name path)
if (TARGET ${module})
return()
endif()
find_library(${module}_LIBRARY_DIR NAMES ${name} NAMES_PER_DIR PATHS ${path}
PATH_SUFFIXES lib
)

message(STATUS "find ${name} location ${${module}_LIBRARY_DIR}")
if ("${${module}_LIBRARY_DIR}" STREQUAL "${module}_LIBRARY_DIR-NOTFOUND")
message(FATAL_ERROR "${name} not found in ${path}")
endif()
add_library(${module} SHARED IMPORTED)
set_target_properties(${module} PROPERTIES
IMPORTED_LOCATION ${${module}_LIBRARY_DIR}
)
endfunction()

+ 0
- 13
cmake/external_libs/eigen.cmake View File

@@ -1,13 +0,0 @@
set(Eigen3_CXXFLAGS "-D_FORTIFY_SOURCE=2 -O2")
set(Eigen3_CFLAGS "-D_FORTIFY_SOURCE=2 -O2")
set(Eigen3_NS "ge_")
graphengine_add_pkg(Eigen3
VER 3.3.7
URL https://gitlab.com/libeigen/eigen/-/archive/3.3.7/eigen-3.3.7.tar.gz
MD5 9e30f67e8531477de4117506fe44669b
CMAKE_OPTION -DBUILD_TESTING=OFF)

find_package(Eigen3 3.3.7 REQUIRED ${GE_FIND_NO_DEFAULT_PATH})
set_property(TARGET Eigen3::Eigen PROPERTY IMPORTED_GLOBAL TRUE)
add_library(graphengine::eigen ALIAS Eigen3::Eigen)
include_directories(${EIGEN3_INCLUDE_DIRS})

+ 38
- 0
cmake/external_libs/gflags.cmake View File

@@ -0,0 +1,38 @@
if (HAVE_GFLAGS)
return()
endif()

include(ExternalProject)
#set(CMAKE_INSTALL_PREFIX ${GE_CODE_DIR}/output)

if ((${CMAKE_INSTALL_PREFIX} STREQUAL /usr/local) OR
(${CMAKE_INSTALL_PREFIX} STREQUAL "C:/Program Files (x86)/ascend"))
set(CMAKE_INSTALL_PREFIX ${GE_CODE_DIR}/output CACHE STRING "path for install()" FORCE)
message(STATUS "No install prefix selected, default to ${CMAKE_INSTALL_PREFIX}.")
endif()

ExternalProject_Add(gflags_build
#URL http://tfk.inhuawei.com/api/containers/container1/download/protobuf-3.8.0.tar.gz
#URL /home/txd/workspace/linux_cmake/pkg/protobuf-3.8.0.tar.gz
SOURCE_DIR ${GE_CODE_DIR}/../third_party/gflags/src/gflags-2.2.2
CONFIGURE_COMMAND ${CMAKE_COMMAND} -DCMAKE_CXX_FLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}/gflags <SOURCE_DIR>
BUILD_COMMAND $(MAKE)
INSTALL_COMMAND $(MAKE) install
EXCLUDE_FROM_ALL TRUE
)

set(GFLAGS_PKG_DIR ${CMAKE_INSTALL_PREFIX}/gflags)

add_library(gflags_static STATIC IMPORTED)

set_target_properties(gflags_static PROPERTIES
IMPORTED_LOCATION ${GFLAGS_PKG_DIR}/lib/libgflags.a
)

add_library(gflags INTERFACE)
target_include_directories(gflags INTERFACE ${GFLAGS_PKG_DIR}/include)
target_link_libraries(gflags INTERFACE gflags_static)

add_dependencies(gflags gflags_build)

set(HAVE_GFLAGS TRUE CACHE BOOL "gflags build add")

+ 0
- 16
cmake/external_libs/gtest.cmake View File

@@ -1,16 +0,0 @@
set(ge_gtest_CXXFLAGS "-D_FORTIFY_SOURCE=2 -O2 -fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack")
set(ge_gtest_CFLAGS "-D_FORTIFY_SOURCE=2 -O2 -fstack-protector-all -Wl,-z,relro,-z,now,-z,noexecstack")

graphengine_add_pkg(ge_gtest
VER 1.8.0
LIBS gtest gtest_main
URL https://github.com/google/googletest/archive/release-1.8.0.tar.gz
MD5 16877098823401d1bf2ed7891d7dce36
CMAKE_OPTION -DBUILD_TESTING=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBUILD_SHARED_LIBS=ON
-DCMAKE_MACOSX_RPATH=TRUE -Dgtest_disable_pthreads=ON)

add_library(graphengine::gtest ALIAS ge_gtest::gtest)
add_library(graphengine::gtest_main ALIAS ge_gtest::gtest_main)
include_directories(${ge_gtest_INC})
file(COPY ${ge_gtest_INC}/../lib/libgtest.so DESTINATION ${CMAKE_SOURCE_DIR}/build/graphengine)
file(COPY ${ge_gtest_INC}/../lib/libgtest_main.so DESTINATION ${CMAKE_SOURCE_DIR}/build/graphengine)

+ 23
- 9
cmake/external_libs/json.cmake View File

@@ -1,9 +1,23 @@
set(nlohmann_json_CXXFLAGS "-D_FORTIFY_SOURCE=2 -O2")
set(nlohmann_json_CFLAGS "-D_FORTIFY_SOURCE=2 -O2")
graphengine_add_pkg(ge_nlohmann_json
VER 3.6.1
HEAD_ONLY ./
URL https://github.com/nlohmann/json/releases/download/v3.6.1/include.zip
MD5 0dc903888211db3a0f170304cd9f3a89)
include_directories(${ge_nlohmann_json_INC})
add_library(graphengine::json ALIAS ge_nlohmann_json)
if (HAVE_JSON)
return()
endif()

include(ExternalProject)

set(JSON_SRC_DIR ${GE_CODE_DIR}/../third_party/json/include)
ExternalProject_Add(json_build
#URL https://github.com/nlohmann/json/releases/download/v3.6.1/include.zip
#URL /home/txd/workspace/cloud_code/pkg/include.zip
SOURCE_DIR ${JSON_SRC_DIR}
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ""
EXCLUDE_FROM_ALL TRUE
)


add_library(json INTERFACE)
target_include_directories(json INTERFACE ${JSON_SRC_DIR})
add_dependencies(json json_build)

set(HAVE_JSON TRUE CACHE BOOL "json build add")

+ 29
- 5
cmake/external_libs/onnx.cmake View File

@@ -1,5 +1,29 @@
graphengine_add_pkg(onnx
VER 1.6.0
HEAD_ONLY ./
URL https://github.com/onnx/onnx/releases/download/v1.6.0/onnx-1.6.0.tar.gz
MD5 512f2779d6215d4a36f366b6b9acdf1e)
include(ExternalProject)

#set(ONNX_SRC_DIR /home/txd/workspace/cloud_code/graphengine/build/graphengine/open_source/onnx)
#set(ONNX_PROTO ${ONNX_SRC_DIR}/onnx/onnx.proto)
set(ONNX_PROTO_DIR ${CMAKE_BINARY_DIR}/onnx)
set(ONNX_PROTO_FILE ${ONNX_PROTO_DIR}/onnx.proto)
file(MAKE_DIRECTORY ${ONNX_PROTO_DIR})

ExternalProject_Add(onnx
#URL https://github.com/onnx/onnx/releases/download/v1.6.0/onnx-1.6.0.tar.gz
URL /home/txd/workspace/cloud_code/pkg/onnx-1.6.0.tar.gz
#URL_HASH SHA256=3b88c3fe521151651a0403c4d131cb2e0311bd28b753ef692020a432a81ce345
#SOURCE_DIR ${ONNX_SRC_DIR}
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
#INSTALL_COMMAND ""
INSTALL_COMMAND ${CMAKE_COMMAND} -E copy <SOURCE_DIR>/onnx/onnx.proto ${ONNX_PROTO_FILE}
#BUILD_ALWAYS TRUE
EXCLUDE_FROM_ALL TRUE
)

macro(onnx_protobuf_generate comp c_var h_var)
add_custom_command(OUTPUT ${ONNX_PROTO_FILE}
DEPENDS onnx
)
ge_protobuf_generate(${comp} ${c_var} ${h_var} ${ONNX_PROTO_FILE})
endmacro()



+ 0
- 54
cmake/external_libs/protobuf.cmake View File

@@ -1,54 +0,0 @@
if (NOT TARGET protobuf::protobuf)
set(protobuf_USE_STATIC_LIBS ON)
set(protobuf_CXXFLAGS "-Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -fstack-protector-all -D_FORTIFY_SOURCE=2 -O2")
set(protobuf_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack")
set(_ge_tmp_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
string(REPLACE " -Wall" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
string(REPLACE " -Werror" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
graphengine_add_pkg(protobuf
VER 3.8.0
LIBS protobuf
EXE protoc
URL https://github.com/protocolbuffers/protobuf/archive/v3.8.0.tar.gz
MD5 3d9e32700639618a4d2d342c99d4507a
CMAKE_PATH ../cmake/
CMAKE_OPTION -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF)
set(CMAKE_CXX_FLAGS ${_ge_tmp_CMAKE_CXX_FLAGS})
endif()
add_library(graphengine::protobuf ALIAS protobuf::protobuf)
set(PROTOBUF_LIBRARY protobuf::protobuf)
include_directories(${protobuf_INC})
include_directories(${protobuf_DIRPATH}/src)

function(ge_protobuf_generate comp c_var h_var)
if(NOT ARGN)
message(SEND_ERROR "Error: ge_protobuf_generate() called without any proto files")
return()
endif()

set(${c_var})
set(${h_var})

foreach(file ${ARGN})
get_filename_component(abs_file ${file} ABSOLUTE)
get_filename_component(file_name ${file} NAME_WE)
get_filename_component(file_dir ${abs_file} PATH)

list(APPEND ${c_var} "${CMAKE_BINARY_DIR}/proto/${comp}/proto/${file_name}.pb.cc")
list(APPEND ${h_var} "${CMAKE_BINARY_DIR}/proto/${comp}/proto/${file_name}.pb.h")

add_custom_command(
OUTPUT "${CMAKE_BINARY_DIR}/proto/${comp}/proto/${file_name}.pb.cc"
"${CMAKE_BINARY_DIR}/proto/${comp}/proto/${file_name}.pb.h"
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/proto/${comp}/proto"
COMMAND protobuf::protoc -I${file_dir} --cpp_out=${CMAKE_BINARY_DIR}/proto/${comp}/proto ${abs_file}
DEPENDS protobuf::protoc ${abs_file}
COMMENT "Running C++ protocol buffer compiler on ${file}" VERBATIM )
endforeach()

set_source_files_properties(${${c_var}} ${${h_var}} PROPERTIES GENERATED TRUE)
set(${c_var} ${${c_var}} PARENT_SCOPE)
set(${h_var} ${${h_var}} PARENT_SCOPE)

endfunction()

+ 59
- 0
cmake/external_libs/protobuf_shared.cmake View File

@@ -0,0 +1,59 @@
if (HAVE_PROTOBUF)
return()
endif()

include(ExternalProject)
include(GNUInstallDirs)

if ((${CMAKE_INSTALL_PREFIX} STREQUAL /usr/local) OR
(${CMAKE_INSTALL_PREFIX} STREQUAL "C:/Program Files (x86)/ascend"))
set(CMAKE_INSTALL_PREFIX ${GE_CODE_DIR}/output CACHE STRING "path for install()" FORCE)
message(STATUS "No install prefix selected, default to ${CMAKE_INSTALL_PREFIX}.")
endif()

set(protobuf_CXXFLAGS "-Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -fstack-protector-all -D_FORTIFY_SOURCE=2 $<$<STREQUAL:${PRODUCT_SIDE},host>:-D_GLIBCXX_USE_CXX11_ABI=0> -O2")
set(protobuf_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack")
ExternalProject_Add(protobuf_build
#URL http://tfk.inhuawei.com/api/containers/container1/download/protobuf-3.8.0.tar.gz
#URL /home/txd/workspace/linux_cmake/pkg/protobuf-3.8.0.tar.gz
#SOURCE_DIR ${GE_CODE_DIR}/third_party/protobuf/src/protobuf-3.8.0
DOWNLOAD_COMMAND ${CMAKE_COMMAND} -E copy_directory ${GE_CODE_DIR}/../third_party/protobuf/src/protobuf-3.8.0 <SOURCE_DIR>
#CONFIGURE_COMMAND ${CMAKE_COMMAND}
#-DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR}
#-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
#-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
#-DCMAKE_LINKER=${CMAKE_LINKER}
#-DCMAKE_AR=${CMAKE_AR}
#-DCMAKE_RANLIB=${CMAKE_RANLIB}
#-Dprotobuf_WITH_ZLIB=OFF
#-Dprotobuf_BUILD_TESTS=OFF -DBUILD_SHARED_LIBS=ON -DCMAKE_CXX_FLAGS=${protobuf_CXXFLAGS} -DCMAKE_CXX_LDFLAGS=${protobuf_LDFLAGS} -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}/protobuf <SOURCE_DIR>/cmake
CONFIGURE_COMMAND cd <SOURCE_DIR>
&& ./autogen.sh && cd <BINARY_DIR> && <SOURCE_DIR>/configure --prefix=${CMAKE_INSTALL_PREFIX}/protobuf --with-zlib=no CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CXXFLAGS=${protobuf_CXXFLAGS} LDFLAGS=${protobuf_LDFLAGS}
&& bash -c "sed -i 's|^hardcode_libdir_flag_spec=.*|hardcode_libdir_flag_spec=\"\"|g' libtool && sed -i 's|^runpath_var=LD_RUN_PATH|runpath_var=DIE_RPATH_DIE|g' libtool"
BUILD_COMMAND $(MAKE)
INSTALL_COMMAND $(MAKE) install
EXCLUDE_FROM_ALL TRUE
)
include(GNUInstallDirs)

set(PROTOBUF_SHARED_PKG_DIR ${CMAKE_INSTALL_PREFIX}/protobuf)

add_library(protobuf SHARED IMPORTED)

file(MAKE_DIRECTORY ${PROTOBUF_SHARED_PKG_DIR}/include)

set_target_properties(protobuf PROPERTIES
IMPORTED_LOCATION ${PROTOBUF_SHARED_PKG_DIR}/lib/libprotobuf.so
)

target_include_directories(protobuf INTERFACE ${PROTOBUF_SHARED_PKG_DIR}/include)

set(INSTALL_BASE_DIR "")
set(INSTALL_LIBRARY_DIR lib)

install(FILES ${PROTOBUF_SHARED_PKG_DIR}/lib/libprotobuf.so ${PROTOBUF_SHARED_PKG_DIR}/lib/libprotobuf.so.19.0.0 OPTIONAL
DESTINATION ${INSTALL_LIBRARY_DIR})

add_dependencies(protobuf protobuf_build)

set(HAVE_PROTOBUF TRUE CACHE BOOL "protobuf build add")

+ 43
- 0
cmake/external_libs/protobuf_static.cmake View File

@@ -0,0 +1,43 @@
include(ExternalProject)
include(GNUInstallDirs)
#set(CMAKE_INSTALL_PREFIX ${GE_CODE_DIR}/output)

if ((${CMAKE_INSTALL_PREFIX} STREQUAL /usr/local) OR
(${CMAKE_INSTALL_PREFIX} STREQUAL "C:/Program Files (x86)/ascend"))
set(CMAKE_INSTALL_PREFIX ${GE_CODE_DIR}/output CACHE STRING "path for install()" FORCE)
message(STATUS "No install prefix selected, default to ${CMAKE_INSTALL_PREFIX}.")
endif()

set(protobuf_CXXFLAGS "-Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -fstack-protector-all -D_FORTIFY_SOURCE=2 $<$<STREQUAL:${PRODUCT_SIDE},host>:-D_GLIBCXX_USE_CXX11_ABI=0> -O2")
set(protobuf_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack")
set(PROTOBUF_STATIC_PKG_DIR ${CMAKE_INSTALL_PREFIX}/protobuf_static)
ExternalProject_Add(protobuf_static_build
#URL http://tfk.inhuawei.com/api/containers/container1/download/protobuf-3.8.0.tar.gz
#URL /home/txd/workspace/linux_cmake/pkg/protobuf-3.8.0.tar.gz
SOURCE_DIR ${GE_CODE_DIR}/../third_party/protobuf/src/protobuf-3.8.0
CONFIGURE_COMMAND ${CMAKE_COMMAND}
-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR}
-DCMAKE_LINKER=${CMAKE_LINKER}
-DCMAKE_AR=${CMAKE_AR}
-DCMAKE_RANLIB=${CMAKE_RANLIB}
-Dprotobuf_WITH_ZLIB=OFF
-Dprotobuf_BUILD_TESTS=OFF -DCMAKE_CXX_FLAGS=${protobuf_CXXFLAGS} -DCMAKE_CXX_LDFLAGS=${protobuf_LDFLAGS} -DCMAKE_INSTALL_PREFIX=${PROTOBUF_STATIC_PKG_DIR} <SOURCE_DIR>/cmake
BUILD_COMMAND $(MAKE)
INSTALL_COMMAND $(MAKE) install
EXCLUDE_FROM_ALL TRUE
)
include(GNUInstallDirs)

add_library(protobuf_static_lib STATIC IMPORTED)

set_target_properties(protobuf_static_lib PROPERTIES
IMPORTED_LOCATION ${PROTOBUF_STATIC_PKG_DIR}/${CMAKE_INSTALL_LIBDIR}/libprotobuf.a
)

add_library(protobuf_static INTERFACE)
target_include_directories(protobuf_static INTERFACE ${PROTOBUF_STATIC_PKG_DIR}/include)
target_link_libraries(protobuf_static INTERFACE protobuf_static_lib)

add_dependencies(protobuf_static protobuf_static_build)

+ 102
- 0
cmake/external_libs/protoc.cmake View File

@@ -0,0 +1,102 @@
if (HAVE_PROTOC)
return()
endif()
include(ExternalProject)
include(GNUInstallDirs)
#set(CMAKE_INSTALL_PREFIX ${GE_CODE_DIR}/output)
if ((${CMAKE_INSTALL_PREFIX} STREQUAL /usr/local) OR
(${CMAKE_INSTALL_PREFIX} STREQUAL "C:/Program Files (x86)/ascend"))
set(CMAKE_INSTALL_PREFIX ${GE_CODE_DIR}/output CACHE STRING "path for install()" FORCE)
message(STATUS "No install prefix selected, default to ${CMAKE_INSTALL_PREFIX}.")
endif()
set(protobuf_CXXFLAGS "-Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -fstack-protector-all -D_FORTIFY_SOURCE=2 -D_GLIBCXX_USE_CXX11_ABI=0 -O2")
set(protobuf_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack")
ExternalProject_Add(protoc_build
#URL http://tfk.inhuawei.com/api/containers/container1/download/protobuf-3.8.0.tar.gz
#URL /home/txd/workspace/linux_cmake/pkg/protobuf-3.8.0.tar.gz
SOURCE_DIR ${GE_CODE_DIR}/../third_party/protobuf/src/protobuf-3.8.0
CONFIGURE_COMMAND ${CMAKE_COMMAND} -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF -DCMAKE_CXX_FLAGS=${protobuf_CXXFLAGS} -DCMAKE_CXX_LDFLAGS=${protobuf_LDFLAGS} -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}/protoc <SOURCE_DIR>/cmake
BUILD_COMMAND $(MAKE)
INSTALL_COMMAND $(MAKE) install
EXCLUDE_FROM_ALL TRUE
)
set(PROTOC_PKG_DIR ${CMAKE_INSTALL_PREFIX}/protoc)
set(protoc_EXECUTABLE ${PROTOC_PKG_DIR}/${CMAKE_INSTALL_BINDIR}/protoc)
function(protobuf_generate comp c_var h_var)
if(NOT ARGN)
message(SEND_ERROR "Error: protobuf_generate() called without any proto files")
return()
endif()
set(${c_var})
set(${h_var})
foreach(file ${ARGN})
get_filename_component(abs_file ${file} ABSOLUTE)
get_filename_component(file_name ${file} NAME_WE)
get_filename_component(file_dir ${abs_file} PATH)
get_filename_component(parent_subdir ${file_dir} NAME)
if("${parent_subdir}" STREQUAL "proto")
set(proto_output_path ${CMAKE_BINARY_DIR}/proto/${comp}/proto)
else()
set(proto_output_path ${CMAKE_BINARY_DIR}/proto/${comp}/proto/${parent_subdir})
endif()
list(APPEND ${c_var} "${proto_output_path}/${file_name}.pb.cc")
list(APPEND ${h_var} "${proto_output_path}/${file_name}.pb.h")
add_custom_command(
OUTPUT "${proto_output_path}/${file_name}.pb.cc" "${proto_output_path}/${file_name}.pb.h"
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
COMMAND ${CMAKE_COMMAND} -E make_directory "${proto_output_path}"
COMMAND ${protoc_EXECUTABLE} -I${file_dir} --cpp_out=${proto_output_path} ${abs_file}
DEPENDS protoc_build ${abs_file}
COMMENT "Running C++ protocol buffer compiler on ${file}" VERBATIM )
endforeach()
set_source_files_properties(${${c_var}} ${${h_var}} PROPERTIES GENERATED TRUE)
set(${c_var} ${${c_var}} PARENT_SCOPE)
set(${h_var} ${${h_var}} PARENT_SCOPE)
endfunction()
function(protobuf_generate_py comp py_var)
if(NOT ARGN)
message(SEND_ERROR "Error: protobuf_generate_py() called without any proto files")
return()
endif()
set(${py_var})
foreach(file ${ARGN})
get_filename_component(abs_file ${file} ABSOLUTE)
get_filename_component(file_name ${file} NAME_WE)
get_filename_component(file_dir ${abs_file} PATH)
get_filename_component(parent_subdir ${file_dir} NAME)
if("${parent_subdir}" STREQUAL "proto")
set(proto_output_path ${CMAKE_BINARY_DIR}/proto/${comp}/proto)
else()
set(proto_output_path ${CMAKE_BINARY_DIR}/proto/${comp}/proto/${parent_subdir})
endif()
list(APPEND ${py_var} "${proto_output_path}/${file_name}_pb2.py")
add_custom_command(
OUTPUT "${proto_output_path}/${file_name}_pb2.py"
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
COMMAND ${CMAKE_COMMAND} -E make_directory "${proto_output_path}"
COMMAND ${protoc_EXECUTABLE} -I${file_dir} --python_out=${proto_output_path} ${abs_file}
DEPENDS protoc_build ${abs_file}
COMMENT "Running PYTHON protocol buffer compiler on ${file}" VERBATIM )
endforeach()
set_source_files_properties(${${py_var}} PROPERTIES GENERATED TRUE)
set(${py_var} ${${py_var}} PARENT_SCOPE)
endfunction()
set(HAVE_PROTOC TRUE CACHE BOOL "protoc build add")

+ 60
- 11
cmake/external_libs/securec.cmake View File

@@ -1,11 +1,60 @@
graphengine_add_pkg(securec
VER 1.1.10
URL https://gitee.com/openeuler/bounds_checking_function/repository/archive/v1.1.10.tar.gz
MD5 0782dd2351fde6920d31a599b23d8c91
LIBS c_sec
PATCHES ${GE_SOURCE_DIR}/third_party/patch/securec/securec.patch001
CMAKE_OPTION "-DCMAKE_BUILD_TYPE=Release"
)
include_directories(${securec_INC})
file(COPY ${securec_INC}/../lib/libc_sec.so DESTINATION ${CMAKE_SOURCE_DIR}/build/graphengine)
add_library(graphengine::securec ALIAS securec::c_sec)
if (HAVE_C_SEC)
return()
endif()

include(ExternalProject)

if ((${CMAKE_INSTALL_PREFIX} STREQUAL /usr/local) OR
(${CMAKE_INSTALL_PREFIX} STREQUAL "C:/Program Files (x86)/ascend"))
set(CMAKE_INSTALL_PREFIX ${GE_CODE_DIR}/output CACHE STRING "path for install()" FORCE)
message(STATUS "No install prefix selected, default to ${CMAKE_INSTALL_PREFIX}.")
endif()

ExternalProject_Add(c_sec_build
#URL http://tfk.inhuawei.com/api/containers/container1/download/protobuf-3.8.0.tar.gz
#URL /home/txd/workspace/linux_cmake/pkg/protobuf-3.8.0.tar.gz
SOURCE_DIR ${GE_CODE_DIR}/../libc_sec
CONFIGURE_COMMAND ${CMAKE_COMMAND}
-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-DCMAKE_LINKER=${CMAKE_LINKER}
-DCMAKE_AR=${CMAKE_AR}
-DCMAKE_RANLIB=${CMAKE_RANLIB}
-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}/c_sec <SOURCE_DIR>
BUILD_COMMAND $(MAKE)
INSTALL_COMMAND $(MAKE) install
EXCLUDE_FROM_ALL TRUE
)

set(C_SEC_PKG_DIR ${CMAKE_INSTALL_PREFIX}/c_sec)

add_library(c_sec SHARED IMPORTED)

file(MAKE_DIRECTORY ${C_SEC_PKG_DIR}/include)

set_target_properties(c_sec PROPERTIES
IMPORTED_LOCATION ${C_SEC_PKG_DIR}/lib/libc_sec.so
)

target_include_directories(c_sec INTERFACE ${C_SEC_PKG_DIR}/include)

add_dependencies(c_sec c_sec_build)

set(INSTALL_BASE_DIR "")
set(INSTALL_LIBRARY_DIR lib)

install(FILES ${C_SEC_PKG_DIR}/lib/libc_sec.so OPTIONAL
DESTINATION ${INSTALL_LIBRARY_DIR})

add_library(c_sec_static_lib STATIC IMPORTED)
set_target_properties(c_sec_static_lib PROPERTIES
IMPORTED_LOCATION ${C_SEC_PKG_DIR}/lib/libc_sec.a
)

add_library(c_sec_static INTERFACE)
target_include_directories(c_sec_static INTERFACE ${C_SEC_PKG_DIR}/include)
target_link_libraries(c_sec_static INTERFACE c_sec_static_lib)

add_dependencies(c_sec_static c_sec_build)

set(HAVE_C_SEC TRUE CACHE BOOL "c_sec build add")

+ 0
- 371
cmake/ge_utils.cmake View File

@@ -1,371 +0,0 @@
include(FetchContent)
set(FETCHCONTENT_QUIET OFF)

function(graphengine_add_submodule_obj des_submodule_objs sub_dir submodule_name_obj)

add_subdirectory(${sub_dir})

if(NOT TARGET ${submodule_name_obj})
message(FATAL_ERROR "Can not find submodule '${submodule_name_obj}'. in ${CMAKE_CURRENT_LIST_FILE}")
endif()
if("$<TARGET_OBJECTS:${submodule_name_obj}>" IN_LIST ${des_submodule_objs})
message(FATAL_ERROR "submodule '${submodule_name_obj}' added more than once. in ${CMAKE_CURRENT_LIST_FILE}")
endif()

set(${des_submodule_objs} ${${des_submodule_objs}} $<TARGET_OBJECTS:${submodule_name_obj}> PARENT_SCOPE)

endfunction()

if (DEFINED ENV{MSLIBS_CACHE_PATH})
set(_MS_LIB_CACHE $ENV{MSLIBS_CACHE_PATH})
else()
set(_MS_LIB_CACHE ${CMAKE_BINARY_DIR}/.mslib)
endif ()
message("MS LIBS CACHE PATH: ${_MS_LIB_CACHE}")

if (NOT EXISTS ${_MS_LIB_CACHE})
file(MAKE_DIRECTORY ${_MS_LIB_CACHE})
endif ()

if (DEFINED ENV{MSLIBS_SERVER})
set(LOCAL_LIBS_SERVER $ENV{MSLIBS_SERVER})
message("LOCAL_LIBS_SERVER: ${LOCAL_LIBS_SERVER}")
endif ()

include(ProcessorCount)
ProcessorCount(N)
if (JOBS)
set(THNUM ${JOBS})
else()
set(JOBS 8)
if (${JOBS} GREATER ${N})
set(THNUM ${N})
endif()
endif ()
message("set make thread num: ${THNUM}")

if(LOCAL_LIBS_SERVER)
if (NOT ENV{no_proxy})
set(ENV{no_proxy} "${LOCAL_LIBS_SERVER}")
else()
string(FIND $ENV{no_proxy} ${LOCAL_LIBS_SERVER} IP_POS)
if (${IP_POS} EQUAL -1)
set(ENV{no_proxy} "$ENV{no_proxy},${LOCAL_LIBS_SERVER}")
endif ()
endif ()
endif()

function(__download_pkg pkg_name pkg_url pkg_md5)

if(LOCAL_LIBS_SERVER)
get_filename_component(_URL_FILE_NAME ${pkg_url} NAME)
set(pkg_url "http://${LOCAL_LIBS_SERVER}:8081/libs/${pkg_name}/${_URL_FILE_NAME}" ${pkg_url})
endif()

FetchContent_Declare(
${pkg_name}
URL ${pkg_url}
URL_HASH MD5=${pkg_md5}
)
FetchContent_GetProperties(${pkg_name})
message("download: ${${pkg_name}_SOURCE_DIR} , ${pkg_name} , ${pkg_url}")
if(NOT ${pkg_name}_POPULATED)
FetchContent_Populate(${pkg_name})
set(${pkg_name}_SOURCE_DIR ${${pkg_name}_SOURCE_DIR} PARENT_SCOPE)
endif()

endfunction()

function(__download_pkg_with_git pkg_name pkg_url pkg_git_commit pkg_md5)

if(LOCAL_LIBS_SERVER)
set(pkg_url "http://${LOCAL_LIBS_SERVER}:8081/libs/${pkg_name}/${pkg_git_commit}")
FetchContent_Declare(
${pkg_name}
URL ${pkg_url}
URL_HASH MD5=${pkg_md5}
)
else()
FetchContent_Declare(
${pkg_name}
GIT_REPOSITORY ${pkg_url}
GIT_TAG ${pkg_git_commit})
endif()
FetchContent_GetProperties(${pkg_name})
message("download: ${${pkg_name}_SOURCE_DIR} , ${pkg_name} , ${pkg_url}")
if(NOT ${pkg_name}_POPULATED)
FetchContent_Populate(${pkg_name})
set(${pkg_name}_SOURCE_DIR ${${pkg_name}_SOURCE_DIR} PARENT_SCOPE)
endif()

endfunction()


function(__find_pkg_then_add_target pkg_name pkg_exe)

unset(${pkg_name}_LIBS)

message("_FIND:${${pkg_name}_BASE_DIR}")

if(pkg_exe)
find_program(${pkg_exe}_EXE ${pkg_exe} PATHS ${${pkg_name}_BASE_DIR}/bin NO_DEFAULT_PATH)
if(NOT ${pkg_exe}_EXE)
return()
endif()
add_executable(${pkg_name}::${pkg_exe} IMPORTED GLOBAL)
set_target_properties(${pkg_name}::${pkg_exe} PROPERTIES
IMPORTED_LOCATION ${${pkg_exe}_EXE}
)
message("found ${${pkg_exe}_EXE}")
endif()

foreach(_LIB_NAME ${ARGN})
set(_LIB_SEARCH_NAME ${_LIB_NAME})
set(_LIB_TYPE SHARED)
if (${pkg_name}_USE_STATIC_LIBS)
set(_LIB_SEARCH_NAME "${CMAKE_STATIC_LIBRARY_PREFIX}${_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(_LIB_TYPE STATIC)
endif ()
set(${_LIB_NAME}_LIB ${_LIB_NAME}_LIB-NOTFOUND)
find_library(${_LIB_NAME}_LIB ${_LIB_SEARCH_NAME} PATHS ${${pkg_name}_BASE_DIR}/lib NO_DEFAULT_PATH)
if(NOT ${_LIB_NAME}_LIB)
return()
endif()
add_library(${pkg_name}::${_LIB_NAME} ${_LIB_TYPE} IMPORTED GLOBAL)
set_target_properties(${pkg_name}::${_LIB_NAME} PROPERTIES
INTERFACE_INCLUDE_DIRECTORIES "${${pkg_name}_BASE_DIR}/include"
IMPORTED_LOCATION ${${_LIB_NAME}_LIB}
)
list(APPEND ${pkg_name}_LIBS ${pkg_name}::${_LIB_NAME})
message("found ${${_LIB_NAME}_LIB}")
STRING( REGEX REPLACE "(.+)/(.+)" "\\1" LIBPATH ${${_LIB_NAME}_LIB})
set(${pkg_name}_LIBPATH ${LIBPATH} CACHE STRING INTERNAL)
endforeach(_LIB_NAME)

set(${pkg_name}_LIBS ${${pkg_name}_LIBS} PARENT_SCOPE)
endfunction()

function(__exec_cmd)
set(options )
set(oneValueArgs WORKING_DIRECTORY)
set(multiValueArgs COMMAND)

cmake_parse_arguments(EXEC "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} )

execute_process(COMMAND ${EXEC_COMMAND}
WORKING_DIRECTORY ${EXEC_WORKING_DIRECTORY}
RESULT_VARIABLE RESULT)
if(NOT RESULT EQUAL "0")
message(FATAL_ERROR "error! when ${EXEC_COMMAND} in ${EXEC_WORKING_DIRECTORY}")
endif()
endfunction()

function(__check_patches pkg_patches)
# check patches
if (PKG_PATCHES)
file(TOUCH ${_MS_LIB_CACHE}/${pkg_name}_patch.md5)
file(READ ${_MS_LIB_CACHE}/${pkg_name}_patch.md5 ${pkg_name}_PATCHES_MD5)

message("patches md5:${${pkg_name}_PATCHES_MD5}")

set(${pkg_name}_PATCHES_NEW_MD5 )
foreach(_PATCH ${PKG_PATCHES})
file(MD5 ${_PATCH} _PF_MD5)
set(${pkg_name}_PATCHES_NEW_MD5 "${${pkg_name}_PATCHES_NEW_MD5},${_PF_MD5}")
endforeach(_PATCH)

if (NOT ${pkg_name}_PATCHES_MD5 STREQUAL ${pkg_name}_PATCHES_NEW_MD5)
set(${pkg_name}_PATCHES ${PKG_PATCHES})
file(REMOVE_RECURSE "${_MS_LIB_CACHE}/${pkg_name}-subbuild")
file(WRITE ${_MS_LIB_CACHE}/${pkg_name}_patch.md5 ${${pkg_name}_PATCHES_NEW_MD5})
message("patches changed : ${${pkg_name}_PATCHES_NEW_MD5}")
endif ()
endif ()
endfunction()

set(GE_FIND_NO_DEFAULT_PATH NO_CMAKE_PATH NO_CMAKE_ENVIRONMENT_PATH NO_SYSTEM_ENVIRONMENT_PATH
NO_CMAKE_BUILDS_PATH NO_CMAKE_PACKAGE_REGISTRY NO_CMAKE_SYSTEM_PATH
NO_CMAKE_SYSTEM_PACKAGE_REGISTRY)
set(GE_FIND_NO_DEFAULT_PATH ${GE_FIND_NO_DEFAULT_PATH} PARENT_SCOPE)

function(graphengine_add_pkg pkg_name )
set(options )
set(oneValueArgs URL MD5 GIT_REPOSITORY GIT_TAG VER EXE DIR HEAD_ONLY CMAKE_PATH)
set(multiValueArgs CMAKE_OPTION LIBS PRE_CONFIGURE_COMMAND CONFIGURE_COMMAND BUILD_OPTION INSTALL_INCS INSTALL_LIBS PATCHES)
cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} )

if (NOT PKG_CMAKE_PATH)
set(PKG_CMAKE_PATH ..)
endif ()

set(__FIND_PKG_NAME ${pkg_name})
string(TOLOWER ${pkg_name} pkg_name)
message("pkg name:${__FIND_PKG_NAME},${pkg_name}")

set(${pkg_name}_PATCHES_HASH )
foreach(_PATCH ${PKG_PATCHES})
file(MD5 ${_PATCH} _PF_MD5)
set(${pkg_name}_PATCHES_HASH "${${pkg_name}_PATCHES_HASH},${_PF_MD5}")
endforeach(_PATCH)

# check options
set(${pkg_name}_CONFIG_TXT
"${CMAKE_CXX_COMPILER_VERSION}-${CMAKE_C_COMPILER_VERSION}
${ARGN} - ${${pkg_name}_USE_STATIC_LIBS}- ${${pkg_name}_PATCHES_HASH}
${${pkg_name}_CXXFLAGS}--${${pkg_name}_CFLAGS}--${${pkg_name}_LDFLAGS}")
string(REPLACE ";" "-" ${pkg_name}_CONFIG_TXT ${${pkg_name}_CONFIG_TXT})
string(MD5 ${pkg_name}_CONFIG_HASH ${${pkg_name}_CONFIG_TXT})

message("${pkg_name} config hash: ${${pkg_name}_CONFIG_HASH}")

set(${pkg_name}_BASE_DIR ${_MS_LIB_CACHE}/${pkg_name}_${${pkg_name}_CONFIG_HASH})
set(${pkg_name}_DIRPATH ${${pkg_name}_BASE_DIR} CACHE STRING INTERNAL)

if(EXISTS ${${pkg_name}_BASE_DIR}/options.txt AND PKG_HEAD_ONLY)
set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/${PKG_HEAD_ONLY} PARENT_SCOPE)
add_library(${pkg_name} INTERFACE)
target_include_directories(${pkg_name} INTERFACE ${${pkg_name}_INC})
return()
endif ()

if(NOT PKG_EXE)
set(PKG_EXE 0)
endif()

set(${__FIND_PKG_NAME}_ROOT ${${pkg_name}_BASE_DIR})
set(${__FIND_PKG_NAME}_ROOT ${${pkg_name}_BASE_DIR} PARENT_SCOPE)

if (PKG_LIBS)
__find_pkg_then_add_target(${pkg_name} ${PKG_EXE} ${PKG_LIBS})
if(${pkg_name}_LIBS)
set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/include PARENT_SCOPE)
message("Found libs: ${${pkg_name}_LIBS}")
return()
endif()
elseif(NOT PKG_HEAD_ONLY)
find_package(${__FIND_PKG_NAME} ${PKG_VER} ${GE_FIND_NO_DEFAULT_PATH})
if (${__FIND_PKG_NAME}_FOUND)
set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/include PARENT_SCOPE)
message("Found pkg: ${__FIND_PKG_NAME}")
return()
endif ()
endif ()

if (NOT PKG_DIR)
if (PKG_GIT_REPOSITORY)
__download_pkg_with_git(${pkg_name} ${PKG_GIT_REPOSITORY} ${PKG_GIT_TAG} ${PKG_MD5})
else()
__download_pkg(${pkg_name} ${PKG_URL} ${PKG_MD5})
endif()
else()
set(${pkg_name}_SOURCE_DIR ${PKG_DIR})
endif ()
file(WRITE ${${pkg_name}_BASE_DIR}/options.txt ${${pkg_name}_CONFIG_TXT})
message("${pkg_name}_SOURCE_DIR : ${${pkg_name}_SOURCE_DIR}")

foreach(_PATCH_FILE ${PKG_PATCHES})
message("patching ${${pkg_name}_SOURCE_DIR} -p1 < ${_PATCH_FILE}")
execute_process(COMMAND patch -p1 INPUT_FILE ${_PATCH_FILE}
WORKING_DIRECTORY ${${pkg_name}_SOURCE_DIR}
RESULT_VARIABLE Result)
if(NOT Result EQUAL "0")
message(FATAL_ERROR "Failed patch: ${_PATCH_FILE}")
endif()
endforeach(_PATCH_FILE)

file(LOCK ${${pkg_name}_BASE_DIR} DIRECTORY GUARD FUNCTION RESULT_VARIABLE ${pkg_name}_LOCK_RET TIMEOUT 600)
if(NOT ${pkg_name}_LOCK_RET EQUAL "0")
message(FATAL_ERROR "error! when try lock ${${pkg_name}_BASE_DIR} : ${${pkg_name}_LOCK_RET}")
endif()

if(${pkg_name}_SOURCE_DIR)
if (PKG_HEAD_ONLY)
file(GLOB ${pkg_name}_SOURCE_SUBDIRS ${${pkg_name}_SOURCE_DIR}/*)
file(COPY ${${pkg_name}_SOURCE_SUBDIRS} DESTINATION ${${pkg_name}_BASE_DIR})
set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/${PKG_HEAD_ONLY} PARENT_SCOPE)
add_library(${pkg_name} INTERFACE)
target_include_directories(${pkg_name} INTERFACE ${${pkg_name}_INC})

elseif (PKG_CMAKE_OPTION)
# in cmake
file(MAKE_DIRECTORY ${${pkg_name}_SOURCE_DIR}/_build)
if (${pkg_name}_CFLAGS)
set(${pkg_name}_CMAKE_CFLAGS "-DCMAKE_C_FLAGS=${${pkg_name}_CFLAGS}")
endif ()
if (${pkg_name}_CXXFLAGS)
set(${pkg_name}_CMAKE_CXXFLAGS "-DCMAKE_CXX_FLAGS=${${pkg_name}_CXXFLAGS}")
endif ()

if (${pkg_name}_LDFLAGS)
if (${pkg_name}_USE_STATIC_LIBS)
#set(${pkg_name}_CMAKE_LDFLAGS "-DCMAKE_STATIC_LINKER_FLAGS=${${pkg_name}_LDFLAGS}")
else()
set(${pkg_name}_CMAKE_LDFLAGS "-DCMAKE_SHARED_LINKER_FLAGS=${${pkg_name}_LDFLAGS}")
endif ()
endif ()

__exec_cmd(COMMAND ${CMAKE_COMMAND} ${PKG_CMAKE_OPTION} -G ${CMAKE_GENERATOR}
${${pkg_name}_CMAKE_CFLAGS} ${${pkg_name}_CMAKE_CXXFLAGS} ${${pkg_name}_CMAKE_LDFLAGS}
-DCMAKE_INSTALL_PREFIX=${${pkg_name}_BASE_DIR} ${PKG_CMAKE_PATH}
WORKING_DIRECTORY ${${pkg_name}_SOURCE_DIR}/_build)

__exec_cmd(COMMAND ${CMAKE_COMMAND} --build . --target install -- -j${THNUM}
WORKING_DIRECTORY ${${pkg_name}_SOURCE_DIR}/_build)

else()
if (${pkg_name}_CFLAGS)
set(${pkg_name}_MAKE_CFLAGS "CFLAGS=${${pkg_name}_CFLAGS}")
endif ()
if (${pkg_name}_CXXFLAGS)
set(${pkg_name}_MAKE_CXXFLAGS "CXXFLAGS=${${pkg_name}_CXXFLAGS}")
endif ()
if (${pkg_name}_LDFLAGS)
set(${pkg_name}_MAKE_LDFLAGS "LDFLAGS=${${pkg_name}_LDFLAGS}")
endif ()
# in configure && make
if (PKG_PRE_CONFIGURE_COMMAND)
__exec_cmd(COMMAND ${PKG_PRE_CONFIGURE_COMMAND}
WORKING_DIRECTORY ${${pkg_name}_SOURCE_DIR})
endif ()

if (PKG_CONFIGURE_COMMAND)
__exec_cmd(COMMAND ${PKG_CONFIGURE_COMMAND}
${${pkg_name}_MAKE_CFLAGS} ${${pkg_name}_MAKE_CXXFLAGS} ${${pkg_name}_MAKE_LDFLAGS}
--prefix=${${pkg_name}_BASE_DIR}
WORKING_DIRECTORY ${${pkg_name}_SOURCE_DIR})
endif ()
set(${pkg_name}_BUILD_OPTION ${PKG_BUILD_OPTION})
if (NOT PKG_CONFIGURE_COMMAND)
set(${pkg_name}_BUILD_OPTION ${${pkg_name}_BUILD_OPTION}
${${pkg_name}_MAKE_CFLAGS} ${${pkg_name}_MAKE_CXXFLAGS} ${${pkg_name}_MAKE_LDFLAGS})
endif ()
# build
__exec_cmd(COMMAND ${CMAKE_MAKE_PROGRAM} ${${pkg_name}_BUILD_OPTION} -j${THNUM}
WORKING_DIRECTORY ${${pkg_name}_SOURCE_DIR})

if (PKG_INSTALL_INCS OR PKG_INSTALL_LIBS)
file(GLOB ${pkg_name}_INSTALL_INCS ${${pkg_name}_SOURCE_DIR}/${PKG_INSTALL_INCS})
file(GLOB ${pkg_name}_INSTALL_LIBS ${${pkg_name}_SOURCE_DIR}/${PKG_INSTALL_LIBS})
file(COPY ${${pkg_name}_INSTALL_INCS} DESTINATION ${${pkg_name}_BASE_DIR}/include)
file(COPY ${${pkg_name}_INSTALL_LIBS} DESTINATION ${${pkg_name}_BASE_DIR}/lib)
else()
__exec_cmd(COMMAND ${CMAKE_MAKE_PROGRAM} install WORKING_DIRECTORY ${${pkg_name}_SOURCE_DIR})
endif ()
endif ()
endif()

if (PKG_LIBS)
__find_pkg_then_add_target(${pkg_name} ${PKG_EXE} ${PKG_LIBS})
set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/include PARENT_SCOPE)
if(NOT ${pkg_name}_LIBS)
message(FATAL_ERROR "Can not find pkg: ${pkg_name}")
endif()
else()
find_package(${__FIND_PKG_NAME} ${PKG_VER} QUIET)
if (${__FIND_PKG_NAME}_FOUND)
set(${pkg_name}_INC ${${pkg_name}_BASE_DIR}/include PARENT_SCOPE)
message("Found pkg: ${${__FIND_PKG_NAME}_LIBRARIES}")
return()
endif ()
endif ()
endfunction()

+ 52
- 0
cmake/intf_pub_android.cmake View File

@@ -0,0 +1,52 @@

add_library(intf_pub INTERFACE)

target_compile_options(intf_pub INTERFACE
-Wall
-fPIC
-fstack-protector-strong
)
target_compile_definitions(intf_pub INTERFACE
$<$<STREQUAL:${PRODUCT_SIDE},host>:_GLIBCXX_USE_CXX11_ABI=0>
$<$<CONFIG:Release>:CFG_BUILD_NDEBUG>
$<$<CONFIG:Debug>:CFG_BUILD_DEBUG>
WIN64=1
LINUX=0
)
target_link_options(intf_pub INTERFACE
-Wl,-z,relro
-Wl,-z,now
-Wl,-z,noexecstack
$<$<CONFIG:Release>:-Wl,--build-id=none>
)
target_link_directories(intf_pub INTERFACE
)

add_library(intf_ccec INTERFACE)
target_compile_options(intf_ccec INTERFACE
-mcpu=cortex-a73
--target=aarch64-linux-android29
--sysroot=${HCC_PATH}/../sysroot
-L${HCC_PATH}/../lib/gcc/aarch64-linux-android/4.9.x
-Wall
-fPIC
-fstack-protector-strong
)
target_compile_definitions(intf_ccec INTERFACE
$<$<STREQUAL:${PRODUCT_SIDE},host>:_GLIBCXX_USE_CXX11_ABI=0>
$<$<CONFIG:Release>:CFG_BUILD_NDEBUG>
$<$<CONFIG:Debug>:CFG_BUILD_DEBUG>
)

target_link_options(intf_ccec INTERFACE
-mcpu=cortex-a73
--target=aarch64-linux-android29
--sysroot=${HCC_PATH}/../sysroot
-L${HCC_PATH}/../lib/gcc/aarch64-linux-android/4.9.x
-Wl,-cce-host-android
-Wl,-z,relro
-Wl,-z,now
-Wl,-z,noexecstack
$<$<CONFIG:Release>:-Wl,--build-id=none>
)


+ 32
- 0
cmake/intf_pub_linux.cmake View File

@@ -0,0 +1,32 @@
if (HAVE_PUB)
return()
endif()

add_library(intf_pub INTERFACE)

target_compile_options(intf_pub INTERFACE
-Wall
-fPIC
$<IF:$<STREQUAL:${CMAKE_SYSTEM_NAME},centos>,-fstack-protector-all,-fstack-protector-strong>
$<$<COMPILE_LANGUAGE:CXX>:-std=c++11>
)
target_compile_definitions(intf_pub INTERFACE
$<$<STREQUAL:${PRODUCT_SIDE},host>:_GLIBCXX_USE_CXX11_ABI=0>
$<$<CONFIG:Release>:CFG_BUILD_NDEBUG>
$<$<CONFIG:Debug>:CFG_BUILD_DEBUG>
WIN64=1
LINUX=0
)
target_link_options(intf_pub INTERFACE
-Wl,-z,relro
-Wl,-z,now
-Wl,-z,noexecstack
$<$<CONFIG:Release>:-Wl,--build-id=none>
)
target_link_directories(intf_pub INTERFACE
)
target_link_libraries(intf_pub INTERFACE
-lpthread
)

set(HAVE_PUB TRUE CACHE BOOL "pub add")

+ 24
- 0
cmake/intf_pub_windows.cmake View File

@@ -0,0 +1,24 @@

add_library(intf_pub INTERFACE)

target_compile_options(intf_pub INTERFACE
-Wall
-fPIC
$<IF:$<STREQUAL:${OS_TYPE},centos>,-fstack-protector-all,-fstack-protector-strong>
$<$<COMPILE_LANGUAGE:CXX>:-std=c++11>
)
target_compile_definitions(intf_pub INTERFACE
$<$<STREQUAL:${PRODUCT_SIDE},host>:_GLIBCXX_USE_CXX11_ABI=0>
OS_TYPE=WIN64
WIN64=1
LINUX=0
$<$<CONFIG:Release>:CFG_BUILD_NDEBUG>
$<$<CONFIG:Debug>:CFG_BUILD_DEBUG>
)
target_link_options(intf_pub INTERFACE
$<$<CONFIG:Release>:-Wl,--build-id=none>
)
target_link_directories(intf_pub INTERFACE
)
target_link_libraries(intf_pub INTERFACE
)

+ 829
- 374
ge/CMakeLists.txt
File diff suppressed because it is too large
View File


+ 0
- 0
ge/README.md View File


+ 58
- 60
ge/analyzer/analyzer.cc View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -40,32 +40,34 @@ const std::string kFilePath = "./";
const std::string kAnalyzeFile = "ge_check_op.json";

const std::string kUnknownShape = "unknownshape";
const std::string kUnsupport = "unsupport";
const std::string kUnsupport = "unsupport";

const std::string kSessionId = "session_id";
const std::string kGraphId = "graph_id";
const std::string kOpInfo = "op_info";
const std::string kGraphId = "graph_id";
const std::string kOpInfo = "op_info";
const std::string kErrorType = "error_type";
const std::string kOpName = "name";
const std::string kOpType = "type";
const std::string kReason = "reason";
const std::string kInput = "input";
const std::string kOutput = "output";
const std::string kShape = "shape";
const std::string kDataType = "data_type";
const std::string kLayout = "layout";
const std::string kResult = "result";
const std::string kOp = "op";

std::map<analyzer::AnalyzeType, std::string> errors_map{{PARSER, "paser_error"},
{INFER_SHAPE, "infer_shape_error"},
{CHECKSUPPORT, "check_support_error"},
{GRAPH_OPTIMIZE, "graph_optimize_error"},
{GRAPH_PARTION, "graph_partion_error"},
{GRAPH_BUILDER, "graph_builder_error"}};
} // namespace

Analyzer *Analyzer::GetInstance() {
const std::string kOpName = "name";
const std::string kOpType = "type";
const std::string kReason = "reason";
const std::string kInput = "input";
const std::string kOutput = "output";
const std::string kShape = "shape";
const std::string kDataType = "data_type";
const std::string kLayout = "layout";
const std::string kResult = "result";
const std::string kOp = "op";

std::map<analyzer::AnalyzeType, std::string> errors_map {
{PARSER, "paser_error"},
{INFER_SHAPE, "infer_shape_error"},
{CHECKSUPPORT, "check_support_error"},
{GRAPH_OPTIMIZE, "graph_optimize_error"},
{GRAPH_PARTION, "graph_partion_error"},
{GRAPH_BUILDER, "graph_builder_error"}
};
}

Analyzer* Analyzer::GetInstance() {
static Analyzer instance;
return &instance;
}
@@ -75,7 +77,7 @@ Status Analyzer::BuildJsonObject(uint64_t session_id, uint64_t graph_id) {
std::lock_guard<std::recursive_mutex> lg(mutex_);
auto iter = graph_infos_.find(session_id);
if (iter == graph_infos_.end()) {
auto p = new (std::nothrow) GraphInfo();
auto p = new(std::nothrow) GraphInfo();
GE_CHECK_NOTNULL(p);
std::shared_ptr<GraphInfo> graph_info(p);
std::map<uint64_t, std::shared_ptr<GraphInfo>> graph_map;
@@ -86,7 +88,7 @@ Status Analyzer::BuildJsonObject(uint64_t session_id, uint64_t graph_id) {
} else {
auto iter1 = (iter->second).find(graph_id);
if (iter1 == (iter->second).end()) {
auto p = new (std::nothrow) GraphInfo();
auto p = new(std::nothrow) GraphInfo();
GE_CHECK_NOTNULL(p);
std::shared_ptr<GraphInfo> graph_info(p);
graph_info->session_id = session_id;
@@ -100,7 +102,14 @@ Status Analyzer::BuildJsonObject(uint64_t session_id, uint64_t graph_id) {
}

ge::Status Analyzer::Initialize() {
ClearHistoryFile();
// Initialize file
string real_path = RealPath(kFilePath.c_str());
if (real_path.empty()) {
GELOGE(FAILED, "File path is invalid.");
return FAILED;
}
json_file_name_ = real_path + "/" + kAnalyzeFile;

return SUCCESS;
}

@@ -174,15 +183,8 @@ ge::Status Analyzer::CreateAnalyzerFile() {
return SUCCESS;
}
GELOGD("start to create analyzer file!");
// Check whether the manifest exists, if not, create it.
string real_path = RealPath(kFilePath.c_str());
if (real_path.empty()) {
GELOGE(FAILED, "File path is invalid.");
return FAILED;
}

std::lock_guard<std::mutex> lg(file_mutex_);
json_file_name_ = real_path + "/" + kAnalyzeFile;
GELOGD("Created analyzer file:[%s]", json_file_name_.c_str());
int fd = open(json_file_name_.c_str(), O_WRONLY | O_CREAT | O_TRUNC, kFileAuthority);
if (fd < 0) {
GELOGE(INTERNAL_ERROR, "Fail to open the file: %s.", json_file_name_.c_str());
@@ -198,25 +200,27 @@ ge::Status Analyzer::CreateAnalyzerFile() {
return SUCCESS;
}

ge::Status Analyzer::SaveAnalyzerDataToFile() {
ge::Status Analyzer::SaveAnalyzerDataToFile(uint64_t session_id, uint64_t graph_id) {
GELOGD("start to save analyze file!");

auto graph_info = GetJsonObject(session_id, graph_id);
GE_CHECK_NOTNULL(graph_info);
if (graph_info->op_info.size() == 0) {
GELOGD("session_id:%lu graph_id:%lu does not owner op info, break it!", session_id, graph_id);
return SUCCESS;
}
std::lock_guard<std::mutex> lg(file_mutex_);
json_file_.open(json_file_name_, std::ios::out);
json_file_.open(json_file_name_, std::ios::app);
if (!json_file_.is_open()) {
GELOGE(FAILED, "analyzer file does not exist[%s]", json_file_name_.c_str());
return PARAM_INVALID;
}

std::lock_guard<std::recursive_mutex> lk(mutex_);
for (auto &ele : graph_infos_) {
for (auto &ele2 : ele.second) {
json jsn;
GraphInfoToJson(jsn, *(ele2.second));
json_file_ << jsn.dump(kJsonDumpLevel) << std::endl;
}
}

json jsn;
GraphInfoToJson(jsn, *graph_info);
json_file_ << jsn.dump(kJsonDumpLevel) << std::endl;
json_file_.close();

return SUCCESS;
}

@@ -237,13 +241,7 @@ ge::Status Analyzer::DoAnalyze(DataInfo &data_info) {
return FAILED;
}
// create json file
status = CreateAnalyzerFile();
if (status != SUCCESS) {
GELOGE(status, "create analyzer file failed!");
return status;
}
// save data to file
return SaveAnalyzerDataToFile();
return CreateAnalyzerFile();
}

ge::Status Analyzer::SaveOpInfo(ge::OpDescPtr desc, DataInfo &data_info,
@@ -256,18 +254,18 @@ ge::Status Analyzer::SaveOpInfo(ge::OpDescPtr desc, DataInfo &data_info,
op_info.error_type = iter->second;
op_info.op_name = desc->GetName();
op_info.op_type = desc->GetType();
op_info.reason = data_info.reason;
op_info.reason = data_info.reason;

for (const auto &ptr : desc->GetAllInputsDescPtr()) {
TensorInfo tensor_info;
tensor_info.shape = ptr->GetShape().GetDims();
tensor_info.shape = ptr->GetShape().GetDims();
tensor_info.d_type = ge::TypeUtils::DataTypeToSerialString(ptr->GetDataType());
tensor_info.layout = ge::TypeUtils::FormatToSerialString(ptr->GetFormat());
op_info.input_info.emplace_back(tensor_info);
}
for (const auto &ptr : desc->GetAllOutputsDescPtr()) {
TensorInfo tensor_info;
tensor_info.shape = ptr->GetShape().GetDims();
tensor_info.shape = ptr->GetShape().GetDims();
tensor_info.d_type = ge::TypeUtils::DataTypeToSerialString(ptr->GetDataType());
tensor_info.layout = ge::TypeUtils::FormatToSerialString(ptr->GetFormat());
op_info.output_info.emplace_back(tensor_info);
@@ -277,13 +275,13 @@ ge::Status Analyzer::SaveOpInfo(ge::OpDescPtr desc, DataInfo &data_info,
return SUCCESS;
}

void Analyzer::TensorInfoToJson(json &j, const TensorInfo &tensor_info) {
void Analyzer::TensorInfoToJson(json& j, const TensorInfo &tensor_info) {
j[kShape] = tensor_info.shape;
j[kDataType] = tensor_info.d_type;
j[kLayout] = tensor_info.layout;
}

void Analyzer::OpInfoToJson(json &j, const OpInfo &op_info) {
void Analyzer::OpInfoToJson(json& j, const OpInfo &op_info) {
j[kErrorType] = op_info.error_type;
j[kOpName] = op_info.op_name;
j[kOpType] = op_info.op_type;
@@ -300,7 +298,7 @@ void Analyzer::OpInfoToJson(json &j, const OpInfo &op_info) {
}
}

void Analyzer::GraphInfoToJson(json &j, const GraphInfo &graph_info) {
void Analyzer::GraphInfoToJson(json& j, const GraphInfo &graph_info) {
GELOGD("start to buff graph info!");
j[kSessionId] = graph_info.session_id;
j[kGraphId] = graph_info.graph_id;
@@ -312,4 +310,4 @@ void Analyzer::GraphInfoToJson(json &j, const GraphInfo &graph_info) {
}
j[kOp] = json_op_infos;
}
} // namespace ge
} // namespace ge

+ 31
- 24
ge/analyzer/analyzer.h View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -33,12 +33,12 @@
namespace ge {
namespace analyzer {
enum AnalyzeType {
PARSER = 0,
INFER_SHAPE = 1,
CHECKSUPPORT = 2,
PARSER = 0,
INFER_SHAPE = 1,
CHECKSUPPORT = 2,
GRAPH_OPTIMIZE = 3,
GRAPH_PARTION = 4,
GRAPH_BUILDER = 5,
GRAPH_PARTION = 4,
GRAPH_BUILDER = 5,
};

struct TensorInfo {
@@ -66,7 +66,8 @@ struct DataInfo {
DataInfo() = default;
~DataInfo() = default;

DataInfo(uint64_t sess, uint64_t graph, AnalyzeType type, ge::NodePtr node, std::string error_info) {
DataInfo(uint64_t sess, uint64_t graph, AnalyzeType type,
ge::NodePtr node, std::string error_info) {
session_id = sess;
graph_id = graph;
analyze_type = type;
@@ -79,10 +80,10 @@ struct DataInfo {
ge::NodePtr node_ptr{nullptr};
std::string reason;
};
} // namespace analyzer
}

class Analyzer {
public:
public:
/**
* @ingroup ge
* @brief: get analyzer instance.
@@ -156,33 +157,39 @@ class Analyzer {
*/
ge::Status DoAnalyze(analyzer::DataInfo &data_info);

/**
* @ingroup ge
* @brief: Buff analyzed data and output to json file
* @param [in]: session id , graph id
* @return: 0: SUCCESS other: FAILED
*/
ge::Status SaveAnalyzerDataToFile(uint64_t session_id, uint64_t graph_id);

Analyzer(const Analyzer &) = delete;
Analyzer &operator=(const Analyzer &) = delete;
Analyzer& operator=(const Analyzer&) = delete;
Analyzer(Analyzer &&) = delete;
Analyzer &operator=(Analyzer &&) = delete;

private:
void TensorInfoToJson(nlohmann::json &j, const analyzer::TensorInfo &tensor_info);
void OpInfoToJson(nlohmann::json &j, const analyzer::OpInfo &op_info);
void GraphInfoToJson(nlohmann::json &j, const analyzer::GraphInfo &graph_info);
Analyzer& operator=(Analyzer &&) = delete;
private:
void TensorInfoToJson(nlohmann::json& j, const analyzer::TensorInfo &tensor_info);
void OpInfoToJson(nlohmann::json& j, const analyzer::OpInfo &op_info);
void GraphInfoToJson(nlohmann::json& j, const analyzer::GraphInfo &graph_info);

ge::Status SaveAnalyzerDataToFile();
ge::Status SaveOpInfo(ge::OpDescPtr desc, analyzer::DataInfo &data_info,
std::shared_ptr<analyzer::GraphInfo> graph_info);
std::shared_ptr<analyzer::GraphInfo> graph_info);

void ClearHistoryFile();
ge::Status CreateAnalyzerFile();

explicit Analyzer(){};
explicit Analyzer() {};
~Analyzer() = default;

private:
private:
std::map<uint64_t, std::map<uint64_t, std::shared_ptr<analyzer::GraphInfo>>> graph_infos_;
std::recursive_mutex mutex_; // protect graph_infos_
std::mutex file_mutex_; // protect json_file_
std::recursive_mutex mutex_; // protect graph_infos_
std::mutex file_mutex_; // protect json_file_
std::ofstream json_file_;
std::string json_file_name_;
std::atomic_bool is_json_file_create_{false};
};
} // namespace ge
#endif // DOMI_ANALYZER_ANANLYZER_H_
} // namespace ge
#endif // DOMI_ANALYZER_ANANLYZER_H_

+ 0
- 72
ge/client/CMakeLists.txt View File

@@ -1,72 +0,0 @@
# Copyright 2019-2020 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================

# libge_client.so
# add all proto files, generate corresponding .h and .cc files
set(CMAKE_CXX_FLAGS "-Wno-unused-variable ${CMAKE_CXX_FLAGS}")
file(GLOB PROTO_LIST RELATIVE ${CMAKE_CURRENT_LIST_DIR}
"../proto/ge_api.proto"
)

file(GLOB PROTO_HEADER_LIST RELATIVE ${CMAKE_CURRENT_LIST_DIR}
"../proto/ge_ir.proto"
"../proto/task.proto"
"../proto/om.proto"
"../proto/insert_op.proto"
)

file(GLOB SRC_LIST RELATIVE ${CMAKE_CURRENT_LIST_DIR}
"ge_api.cc"
"ge_prof.cc"
)

ge_protobuf_generate(ge PROTO_SRCS PROTO_HDRS ${PROTO_LIST})
ge_protobuf_generate(ge PROTO_HEADER_SRCS PROTO_HEADER_HDRS ${PROTO_HEADER_LIST})

# include directories
include_directories(${CMAKE_CURRENT_LIST_DIR})
include_directories(${GE_SOURCE_DIR}/ge)
include_directories(${GE_SOURCE_DIR}/inc)
include_directories(${GE_SOURCE_DIR}/inc/external)
include_directories(${GE_SOURCE_DIR}/inc/common)
include_directories(${GE_SOURCE_DIR}/inc/framework)
include_directories(${GE_SOURCE_DIR}/metadef/inc)
include_directories(${GE_SOURCE_DIR}/metadef/inc/external/graph)
include_directories(${GE_SOURCE_DIR}/metadef/inc/graph)
include_directories(${GE_SOURCE_DIR}/third_party/fwkacllib/inc)
include_directories(${GE_SOURCE_DIR}/third_party/fwkacllib/inc/cce)
include_directories(${CMAKE_BINARY_DIR})
include_directories(${CMAKE_BINARY_DIR}/proto/ge)

############ libge_client.so ################
add_library(ge_client SHARED ${SRC_LIST} ${PROTO_SRCS} ${PROTO_HEADER_HDRS})
target_compile_definitions(ge_client PRIVATE
Werror
PROTOBUF_INLINE_NOT_IN_HEADERS=0
REUSE_MEMORY=1
PLATFORM_CLOUD)
target_link_libraries(ge_client
graph
ge_compiler
ge_common
${PROTOBUF_LIBRARY}
${register}
${c_sec}
${slog}
${mmpa}
${runtime}
${msprof}
rt
dl)

+ 1
- 1
ge/client/ge_api.cc View File

@@ -380,7 +380,7 @@ Status Session::RunGraphAsync(uint32_t graph_id, const std::vector<InputTensorIn
}
GELOGT(TRACE_RUNNING, "Run Graph Asynchronously");
GELOGW(
"The callback function will not be checked. Please ensure that the implementation of the function is trusted.");
"The callback function will not be checked. Please ensure that the implementation of the function is trusted.");

Status ret = ge::GELib::GetInstance()->SessionManagerObj().RunGraphAsync(sessionId_, graph_id, inputs, callback);
if (ret != SUCCESS) {


+ 31
- 50
ge/client/ge_prof.cc View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -29,39 +29,22 @@ using std::vector;

namespace {
const uint32_t kMaxDeviceNum = 64;
const std::string PROFILING_INIT = "prof_init";
const std::string PROFILING_FINALIZE = "prof_finalize";
const std::string PROFILING_START = "prof_start";
const std::string PROFILING_STOP = "prof_stop";
const std::string DEVICES_NUMS = "devNums";
const std::string DEVICE_ID_LIST = "devIdList";
const std::string AICORE_METRICS = "aicoreMetrics";
const uint32_t kDeviceListIndex = 3;
const std::string kProfilingInit = "prof_init";
const std::string kProfilingFinalize = "prof_finalize";
const std::string kProfilingStart = "prof_start";
const std::string kProfilingStop = "prof_stop";
const std::string kDeviceNums = "devNums";
const std::string kDeviceIdList = "devIdList";
const std::string kAicoreMetrics = "aicoreMetrics";

const std::map<ge::ProfilingAicoreMetrics, std::string> kProfAicoreMetricsToString = {
{ge::kAicoreArithmaticThroughput, "AICORE_ARITHMATIC_THROUGHPUT"},
{ge::kAicorePipeline, "AICORE_PIPELINE"},
{ge::kAicoreSynchronization, "AICORE_SYNCHRONIZATION"},
{ge::kAicoreMemory, "AICORE_MEMORY"},
{ge::kAicoreInternalMemory, "AICORE_INTERNAL_MEMORY"},
{ge::kAicoreStall, "AICORE_STALL"},
{ge::kAicoreMetricsAll, "AICORE_METRICS_ALL"}};

const std::map<uint64_t, uint64_t> kDataTypeConfigMapping = {{ge::kProfAcl, PROF_ACL_API},
{ge::kProfTaskTime, PROF_TASK_TIME},
{ge::kProfAiCoreMetrics, PROF_AICORE_METRICS},
{ge::kProfAicpuTrace, PROF_AICPU_TRACE},
{ge::kProfModelExecute, PROF_MODEL_EXECUTE},
{ge::kProfRuntimeApi, PROF_RUNTIME_API},
{ge::kProfRuntimeTrace, PROF_RUNTIME_TRACE},
{ge::kProfScheduleTimeline, PROF_SCHEDULE_TIMELINE},
{ge::kProfScheduleTrace, PROF_SCHEDULE_TRACE},
{ge::kProfAiVectorCoreMetrics, PROF_AIVECTORCORE_METRICS},
{ge::kProfSubtaskTime, PROF_SUBTASK_TIME},
{ge::kProfTrainingTrace, PROF_TRAINING_TRACE},
{ge::kProfHcclTrace, PROF_HCCL_TRACE},
{ge::kProfDataProcess, PROF_DATA_PROCESS},
{ge::kProfTaskTrace, PROF_TASK_TRACE},
{ge::kProfModelLoad, PROF_MODEL_LOAD}};
{ge::kAicoreArithmaticThroughput, "AICORE_ARITHMATIC_THROUGHPUT"},
{ge::kAicorePipeline, "AICORE_PIPELINE"},
{ge::kAicoreSynchronization, "AICORE_SYNCHRONIZATION"},
{ge::kAicoreMemory, "AICORE_MEMORY"},
{ge::kAicoreInternalMemory, "AICORE_INTERNAL_MEMORY"},
{ge::kAicoreStall, "AICORE_STALL"}};
} // namespace

static bool g_graph_prof_init_ = false;
@@ -107,11 +90,11 @@ Status aclgrphProfInit(const char *profiler_path, uint32_t length) {
GraphLoader graph_loader;
Command command;
command.cmd_params.clear();
command.cmd_type = PROFILING_INIT;
command.module_index = kProfModelLoad | kProfTrainingTrace;
command.cmd_type = kProfilingInit;
command.module_index = PROF_MODEL_LOAD;
ret = graph_loader.CommandHandle(command);
if (ret != SUCCESS) {
GELOGE(ret, "Handle profiling command %s failed, config = %s", PROFILING_INIT.c_str(), profiler_path);
GELOGE(ret, "Handle profiling command %s failed, config = %s", kProfilingInit.c_str(), profiler_path);
return ret;
}
if (!g_graph_prof_init_) {
@@ -143,10 +126,10 @@ Status aclgrphProfFinalize() {
GraphLoader graph_loader;
Command command;
command.cmd_params.clear();
command.cmd_type = PROFILING_FINALIZE;
command.cmd_type = kProfilingFinalize;
Status ret = graph_loader.CommandHandle(command);
if (ret != SUCCESS) {
GELOGE(ret, "Handle profiling command %s failed.", PROFILING_FINALIZE.c_str());
GELOGE(ret, "Handle profiling command %s failed.", kProfilingFinalize.c_str());
return ret;
}

@@ -164,9 +147,9 @@ Status aclgrphProfFinalize() {

bool TransProfConfigToParam(const aclgrphProfConfig *profiler_config, vector<string> &prof_config_params) {
prof_config_params.clear();
prof_config_params.emplace_back(DEVICES_NUMS);
prof_config_params.emplace_back(kDeviceNums);
prof_config_params.emplace_back(std::to_string(profiler_config->config.devNums));
prof_config_params.emplace_back(DEVICE_ID_LIST);
prof_config_params.emplace_back(kDeviceIdList);
std::string devID = "";
if (profiler_config->config.devNums == 0) {
GELOGW("The device num is invalid.");
@@ -180,9 +163,9 @@ bool TransProfConfigToParam(const aclgrphProfConfig *profiler_config, vector<str
}

prof_config_params.push_back(devID);
prof_config_params.push_back(AICORE_METRICS);
prof_config_params.push_back(kAicoreMetrics);
auto iter =
kProfAicoreMetricsToString.find(static_cast<ProfilingAicoreMetrics>(profiler_config->config.aicoreMetrics));
kProfAicoreMetricsToString.find(static_cast<ProfilingAicoreMetrics>(profiler_config->config.aicoreMetrics));
if (iter == kProfAicoreMetricsToString.end()) {
GELOGW("The prof aicore metrics is invalid.");
return false;
@@ -250,13 +233,7 @@ aclgrphProfConfig *aclgrphProfCreateConfig(uint32_t *deviceid_list, uint32_t dev
}

config->config.aicoreMetrics = static_cast<ProfAicoreMetrics>(aicore_metrics);
uint64_t data_type = 0;
for (auto &iter : kDataTypeConfigMapping) {
if ((iter.first & data_type_config) == iter.first) {
data_type |= iter.second;
}
}
config->config.dataTypeConfig = data_type;
config->config.dataTypeConfig = data_type_config;
GELOGI("Successfully create prof config.");
return config;
}
@@ -309,9 +286,11 @@ Status aclgrphProfStart(aclgrphProfConfig *profiler_config) {
GraphLoader graph_loader;
Command command;
command.cmd_params.clear();
command.cmd_type = PROFILING_START;
command.cmd_type = kProfilingStart;
command.cmd_params = prof_params;
command.module_index = profiler_config->config.dataTypeConfig;
GELOGI("Profiling will start, device nums:%s , deviceID:[%s], data type config: 0x%llx", prof_params[0].c_str(),
prof_params[kDeviceListIndex].c_str(), command.module_index);
ret = graph_loader.CommandHandle(command);
if (ret != SUCCESS) {
GELOGE(ret, "Handle profiling command failed");
@@ -360,9 +339,11 @@ Status aclgrphProfStop(aclgrphProfConfig *profiler_config) {
GraphLoader graph_loader;
Command command;
command.cmd_params.clear();
command.cmd_type = PROFILING_STOP;
command.cmd_type = kProfilingStop;
command.cmd_params = prof_params;
command.module_index = profiler_config->config.dataTypeConfig;
GELOGI("Profiling will stop, device nums:%s , deviceID:[%s], data type config: 0x%llx", prof_params[0].c_str(),
prof_params[kDeviceListIndex].c_str(), command.module_index);
ret = graph_loader.CommandHandle(command);
if (ret != SUCCESS) {
GELOGE(ret, "Handle profiling command failed");


+ 1
- 0
ge/client/proto/ge_api.proto View File

@@ -0,0 +1 @@
../../proto/ge_api.proto

+ 1
- 0
ge/client/proto/ge_ir.proto View File

@@ -0,0 +1 @@
../../proto/ge_ir.proto

+ 1
- 0
ge/client/proto/insert_op.proto View File

@@ -0,0 +1 @@
../../proto/insert_op.proto

+ 396
- 0
ge/client/proto/om.proto View File

@@ -0,0 +1,396 @@
/* Copyright (C) 2018. Huawei Technologies Co., Ltd. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the Apache License Version 2.0.You may not use this file except in compliance with the License.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* Apache License for more details at
* http://www.apache.org/licenses/LICENSE-2.0
*/
syntax = "proto3";

package domi;

enum TargetType
{
MINI = 0;
TINY = 1;
LITE = 2;
}

// offline model
message ModelDef {
string name = 1;
uint32 version = 2;

uint64 memory_size = 10;
uint32 stream_num = 11;
uint32 event_num = 12;
uint64 weight_size = 13;
uint32 label_num = 15;
repeated OpDef op = 20;
TargetType target_type = 23;

map<string, AttrDef> attr = 30;
};

// operator define
message OpDef {
string name = 1;
string type = 2;

uint32 id = 3;
uint32 stream_id = 4;

repeated string input_name = 5;

repeated string src_name = 8;
repeated int32 src_index = 9;
repeated int64 input = 10;
repeated int64 output = 11;
repeated TensorDescriptor input_desc = 12;
repeated TensorDescriptor output_desc = 13;
repeated WeightDef weights = 14;
repeated string dst_name = 15;
repeated int32 dst_index = 16;

repeated int64 workspace = 20;
repeated uint32 workspace_bytes = 21;

repeated string weight_name = 22;
repeated bool is_input_const = 23;

map<string, AttrDef> attr = 30;

QuantizeFactorParams quantize_factor = 31;

oneof op_params {
// start at 100 here
SendOpParams sender_param = 100;
RecvOpParams receiver_param = 200;
ConvolutionOpParams convolution_param = 300;
PoolingOpParams pooling_param = 400;
EltwiseOpParams eltwise_param = 500;
BatchNormOpParams batchnorm_param = 600;
ScaleOpParams scale_param = 700;
FullConnectionOpParams full_connection_param = 800;
SoftmaxOpParams softmax_param = 900;
ActivationOpParams activation_param = 1000;
ReshapeOpParams reshape_param = 1100;
}
};

message SendOpParams {
uint32 event_id = 1;
};

message RecvOpParams {
uint32 event_id = 1;
};

enum QuantizeScaleType
{
VECTOR_SCALE = 0;
SCALAR_SCALE = 1;
}

enum QuantizeScaleMode
{
NORMAL_MODE = 0;
SQRT_MODE = 1;
}

enum QuantizeAlgorithm
{
NON_OFFSET_ALGO = 0;
HALF_OFFSET_ALGO = 1;
ALL_OFFSET_ALGO = 2;
}
message QuantizeFactor
{
QuantizeScaleMode scale_mode = 1;
bytes scale_value = 2;
int64 scale_offset = 3;
bytes offset_data_value = 4;
int64 offset_data_offset = 5;
bytes offset_weight_value = 6;
int64 offset_weight_offset = 7;
bytes offset_pad_value = 8;
int64 offset_pad_offset = 9;
};

message QuantizeCalcFactor
{
bytes offsetw = 1;
int64 offsetw_offset = 2;
bytes offsetd = 3;
int64 offsetd_offset = 4;
bytes scalereq = 5;
int64 scaledreq_offset = 6;
bytes offsetdnext = 7;
int64 offsetdnext_offset = 8;
}

message QuantizeFactorParams
{
QuantizeAlgorithm quantize_algo = 1;
QuantizeScaleType scale_type = 2;
QuantizeFactor quantize_param = 3;
QuantizeFactor dequantize_param = 4;
QuantizeFactor requantize_param = 5;
QuantizeCalcFactor quantizecalc_param = 6;
};

message ConvolutionOpParams {
int32 mode = 1;
int32 algo = 2;
int32 pad_mode = 3;
uint32 group = 4;
uint32 num_output = 5;

repeated uint32 pad = 10;
repeated uint32 stride = 11;
repeated uint32 dilation = 12;
repeated uint32 kernel = 13;

float alpha = 20;
float beta = 21;

WeightDef filter = 40;
WeightDef bias = 41;

bool relu_flag = 62;
repeated uint32 adj = 70;
repeated uint32 target_shape = 71;
repeated uint32 before_pad = 72;
};

message PoolingOpParams {
int32 mode = 1;
int32 nan_opt = 2;
int32 pad_mode = 3;
bool global_pooling = 4;

repeated uint32 window = 10;
repeated uint32 pad = 11;
repeated uint32 stride = 12;
bool ceil_mode = 13;
int32 data_mode = 14;

float alpha = 20;
float beta = 21;
repeated uint32 before_pad = 22;
};

message EltwiseOpParams {
int32 mode = 1;
repeated float coeff = 2;
float alpha = 3;
float beta = 4;
repeated WeightDef weight = 5;
bool relu_flag = 6;
};

message ActivationOpParams {
int32 mode = 1;
float coef = 2;
float alpha = 3;
float beta = 4;
};

message BatchNormOpParams {
int32 mode = 1;

float alpha = 2;
float beta = 3;
double epsilon = 4;//optinal,[default = 1e-5]
bool use_global_stats = 5; //optinal,by default true,testing mode
float moving_average_fraction = 6; //optinal,[default = .999];

WeightDef estimated_mean = 7;
WeightDef estimated_variance = 8;

WeightDef scale = 9;
WeightDef bias = 10;
};

message ScaleOpParams {
WeightDef scale = 1;
WeightDef bias = 2;
};

message ReshapeOpParams {
float alpha = 1;
float beta = 2;
ShapeDef shape = 3;
int32 axis = 4;
int32 num_axes = 5;
int32 format = 6;
};

message SoftmaxOpParams {
int32 algo = 1;
int32 mode = 2;
float alpha = 3;
float beta = 4;
};

message FullConnectionOpParams {
WeightDef filter = 1;
WeightDef bias = 2;
uint32 num_output = 3;
bool relu_flag = 12;
};

message FlattenOpParams {
float alpha = 1;
float beta = 2;
int32 start_axis = 3;
int32 end_axis = 4;
}

message AddLimitedOpParams {
float alpha = 1;
float beta = 2;
int32 axis = 3;
bool broadcast = 4;

repeated WeightDef weight = 10;
};

message MulLimitedOpParams {
float alpha = 1;
float beta = 2;
int32 axis = 3;
bool broadcast = 4;

repeated WeightDef weight = 10;
};

message AddOpParams {
float alpha = 1;
float beta = 2;

repeated WeightDef weight = 10;
};

message MulOpParams {
float alpha = 1;
float beta = 2;

repeated WeightDef weight = 10;
};

message SubOpParams {
float alpha = 1;
float beta = 2;

repeated WeightDef weight = 10;
};

message BiasAddOpParams {
float alpha = 1;
float beta = 2;

WeightDef bias = 10;
};

message MatMulOpParams {
float alpha = 1;
float beta = 2;
bool transposeX = 3;
bool transposeW = 4;

WeightDef filter = 10;
WeightDef bias = 12;
};

message RsqrtOpParams {
float alpha = 1;
float beta = 2;
};


message WeightDef {
int32 format = 1;
int32 data_type = 2;
ShapeDef shape = 3;
bytes data = 4;
int64 data_offset = 5;
uint32 cmps_size = 6;
bytes cmps_tab = 7;
int64 cmps_tab_offset = 10;
CompressInfo cmps_info = 8;
AllOffsetQuantizeInfo alloffset_quantize_info = 11;
}

message ShapeDef {
repeated int64 dim = 1;
}

enum DeviceType {
NPU = 0; // In default, we will use NPU.
CPU = 1; // CPU
}

message AllOffsetQuantizeInfo {
float scale = 1;
int32 offset = 2;
}

message TensorDescriptor {
int32 format = 1;
int32 data_type = 2;
repeated int64 dim = 3;
uint32 size = 4;
bool reuse_input = 5;
bool output_tensor = 7;
DeviceType device_type = 8;
bool input_tensor = 9;
uint32 real_dim_cnt = 10;
uint32 reuse_input_index = 11;
AllOffsetQuantizeInfo alloffset_quantize_info = 12;
}

message CompressInfo {
int32 blockRow = 1; // block row
int32 blockCol = 2; // block col
int32 fractalK = 3; // fractal K
int32 fractalN = 4; // fractal N
int32 lastFractalK = 5; // K of last fractal
int32 lastFractalN = 6; // N of last fractal
int32 cubeSize = 7; // cube's length
int32 loadDir = 8; // data load directtiono 0:col load 1:row load
}

message AttrDef {
message ListValue {
repeated string s = 2; // "list(string)"
repeated int64 i = 3 [packed = true]; // "list(int)"
repeated float f = 4 [packed = true]; // "list(float)"
repeated bool b = 5 [packed = true]; // "list(bool)"
repeated uint32 u = 6 [packed = true]; // "list(uint)"
repeated bytes bt = 7;
}

oneof value {
string s = 2; // "string"
int64 i = 3; // "int"
float f = 4; // "float"
bool b = 5; // "bool"
uint32 u = 6; // "uint32"
bytes bt = 7;
ListValue list = 1; // any "list(...)"
NamedAttrs func = 10;
}
}

// A list of attr names and their values. The whole list is attached
// with a string name. E.g., MatMul[T=float].
message NamedAttrs {
string name = 1;
map<string, AttrDef> attr = 2;
}


+ 1
- 0
ge/client/proto/task.proto View File

@@ -0,0 +1 @@
../../proto/task.proto

+ 162
- 100
ge/common/CMakeLists.txt View File

@@ -1,105 +1,167 @@
# Copyright 2019-2020 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
set(PROTO_LIST
"${METADEF_DIR}/proto/om.proto"
"${METADEF_DIR}/proto/ge_ir.proto"
"${METADEF_DIR}/proto/insert_op.proto"
"${METADEF_DIR}/proto/task.proto"
"${METADEF_DIR}/proto/tensorflow/attr_value.proto"
"${METADEF_DIR}/proto/tensorflow/function.proto"
"${METADEF_DIR}/proto/tensorflow/graph.proto"
"${METADEF_DIR}/proto/tensorflow/node_def.proto"
"${METADEF_DIR}/proto/tensorflow/op_def.proto"
"${METADEF_DIR}/proto/tensorflow/resource_handle.proto"
"${METADEF_DIR}/proto/tensorflow/tensor.proto"
"${METADEF_DIR}/proto/tensorflow/tensor_shape.proto"
"${METADEF_DIR}/proto/tensorflow/types.proto"
"${METADEF_DIR}/proto/tensorflow/versions.proto"
)

# libge_common.so
file(GLOB PROTO_LIST RELATIVE ${CMAKE_CURRENT_LIST_DIR}
"${GE_SOURCE_DIR}/metadef/proto/om.proto"
"${GE_SOURCE_DIR}/metadef/proto/ge_ir.proto"
"${GE_SOURCE_DIR}/metadef/proto/task.proto"
"${GE_SOURCE_DIR}/metadef/proto/insert_op.proto"
)
protobuf_generate(ge PROTO_SRCS PROTO_HDRS ${PROTO_LIST})

file(GLOB SRC_LIST RELATIVE ${CMAKE_CURRENT_LIST_DIR}
"../model/ge_model.cc"
"auth/file_saver.cc"
"context/ctx.cc"
"cust_aicpu_kernel_store.cc"
"debug/memory_dumper.cc"
"dump/dump_properties.cc"
"fmk_error_codes.cc"
"formats/format_transfers/datatype_transfer.cc"
"formats/format_transfers/format_transfer_c1hwncoc0_hwcn.cc"
"formats/format_transfers/format_transfer_dhwcn_fracz3D.cc"
"formats/format_transfers/format_transfer_dhwnc_fracz3D_transpose.cc"
"formats/format_transfers/format_transfer_fractal_nz.cc"
"formats/format_transfers/format_transfer_fractal_z.cc"
"formats/format_transfers/format_transfer_fractal_zz.cc"
"formats/format_transfers/format_transfer_fracz_hwcn.cc"
"formats/format_transfers/format_transfer_fracz_nchw.cc"
"formats/format_transfers/format_transfer_fracz_nhwc.cc"
"formats/format_transfers/format_transfer_hwcn_c1hwncoc0.cc"
"formats/format_transfers/format_transfer_nc1hwc0_nchw.cc"
"formats/format_transfers/format_transfer_nc1hwc0_nhwc.cc"
"formats/format_transfers/format_transfer_nchw_fz_c04.cc"
"formats/format_transfers/format_transfer_nchw_nc1hwc0.cc"
"formats/format_transfers/format_transfer_nhwc_nc1hwc0.cc"
"formats/format_transfers/format_transfer_transpose.cc"
"formats/formats.cc"
"formats/utils/formats_trans_utils.cc"
"fp16_t.cc"
"ge/datatype_util.cc"
"ge/tbe_plugin_manager.cc"
"ge_format_util.cc"
"helper/model_helper.cc"
"helper/om_file_helper.cc"
"kernel_store.cc"
"math/fp16_math.cc"
"model_parser/base.cc"
"model_saver.cc"
"op/attr_value_util.cc"
"op/ge_op_utils.cc"
"properties_manager.cc"
"tbe_kernel_store.cc"
"thread_pool.cc"
"types.cc"
"util.cc"
)
set(SRC_LIST
"context/ctx.cc"
"model_saver.cc"
"ge/datatype_util.cc"
"helper/om_file_helper.cc"
"helper/model_helper.cc"
"../model/ge_model.cc"
"auth/file_saver.cc"
"fp16_t.cc"
"math/fp16_math.cc"
"debug/memory_dumper.cc"
"formats/utils/formats_trans_utils.cc"
"dump/dump_properties.cc"
"formats/format_transfers/datatype_transfer.cc"
"formats/format_transfers/format_transfer_transpose.cc"
"formats/format_transfers/format_transfer_nchw_nc1hwc0.cc"
"formats/format_transfers/format_transfer_fractal_z.cc"
"formats/format_transfers/format_transfer_fractal_nz.cc"
"formats/format_transfers/format_transfer_fractal_zz.cc"
"formats/format_transfers/format_transfer_nhwc_nc1hwc0.cc"
"formats/format_transfers/format_transfer_nc1hwc0_nchw.cc"
"formats/format_transfers/format_transfer_nc1hwc0_nhwc.cc"
"formats/format_transfers/format_transfer_hwcn_c1hwncoc0.cc"
"formats/format_transfers/format_transfer_c1hwncoc0_hwcn.cc"
"formats/format_transfers/format_transfer_fracz_nchw.cc"
"formats/format_transfers/format_transfer_fracz_nhwc.cc"
"formats/format_transfers/format_transfer_fracz_hwcn.cc"
"formats/format_transfers/format_transfer_dhwcn_fracz3D.cc"
"formats/format_transfers/format_transfer_dhwnc_fracz3D_transpose.cc"
"formats/format_transfers/format_transfer_nchw_fz_c04.cc"
"formats/formats.cc"
"ge_format_util.cc"
"fmk_error_codes.cc"
"util.cc"
"properties_manager.cc"
"types.cc"
"model_parser/base.cc"
"kernel_store.cc"
"tbe_kernel_store.cc"
"cust_aicpu_kernel_store.cc"
"op/attr_value_util.cc"
"op/ge_op_utils.cc"
"thread_pool.cc"
"ge/tbe_plugin_manager.cc"
)

ge_protobuf_generate(ge PROTO_SRCS PROTO_HDRS ${PROTO_LIST})
############ libge_common.so ############
add_library(ge_common SHARED ${SRC_LIST} ${PROTO_HDRS})
target_compile_definitions(ge_common PRIVATE
PROTOBUF_INLINE_NOT_IN_HEADERS=0
HOST_VISIBILITY
FMK_SUPPORT_DUMP
OS_CENTOS
)

# include directories
include_directories(${CMAKE_CURRENT_LIST_DIR})
include_directories(${CMAKE_CURRENT_LIST_DIR}/op)
include_directories(${GE_SOURCE_DIR}/ge)
include_directories(${GE_SOURCE_DIR}/inc)
include_directories(${GE_SOURCE_DIR}/inc/common/util)
include_directories(${GE_SOURCE_DIR}/inc/external)
include_directories(${GE_SOURCE_DIR}/inc/framework)
include_directories(${GE_SOURCE_DIR}/metadef/inc)
include_directories(${GE_SOURCE_DIR}/metadef/inc/external)
include_directories(${GE_SOURCE_DIR}/metadef/inc/external/graph)
include_directories(${GE_SOURCE_DIR}/metadef/inc/graph)
include_directories(${GE_SOURCE_DIR}/third_party/fwkacllib/inc)
include_directories(${GE_SOURCE_DIR}/third_party/fwkacllib/inc/cce)
include_directories(${CMAKE_BINARY_DIR})
include_directories(${CMAKE_BINARY_DIR}/proto/ge)
target_compile_options(ge_common PRIVATE
-fvisibility=hidden
-O2
-Werror
)

############ libge_common.so ################
add_library(ge_common SHARED ${SRC_LIST} ${PROTO_HDRS})
target_compile_definitions(ge_common PUBLIC
PROTOBUF_INLINE_NOT_IN_HEADERS=0
HOST_VISIBILITY
OS_CENTOS)
target_link_libraries(ge_common
graph
${PROTOBUF_LIBRARY}
${register}
${c_sec}
${slog}
${mmpa}
${resource}
${error_manager}
rt
dl)
target_include_directories(ge_common PRIVATE
${GE_CODE_DIR}/ge
${GE_CODE_DIR}/ge/common
${GE_CODE_DIR}/ge/common/op
${GE_CODE_DIR}/inc/external
${GE_CODE_DIR}/inc
${GE_CODE_DIR}/inc/framework
${METADEF_DIR}/inc
${METADEF_DIR}/inc/external
${METADEF_DIR}/inc/external/graph
${METADEF_DIR}/inc/graph
${CMAKE_BINARY_DIR}
${CMAKE_BINARY_DIR}/proto/ge
#### yellow zone ####
${GE_DEPEND_DIR}/inc
${GE_DEPEND_DIR}/inc/cce
#### blue zone ####
#${GE_DEPEND_DIR}/include
)

target_link_libraries(ge_common PRIVATE
$<BUILD_INTERFACE:intf_pub>
-Wl,--no-as-needed
graph
protobuf
register
c_sec
error_manager
slog
mmpa
-Wl,--as-needed
json
-lrt
-ldl
)

############ libge_common.a ############
add_library(ge_common_static STATIC ${SRC_LIST} ${PROTO_HDRS})
target_compile_definitions(ge_common_static PRIVATE
PROTOBUF_INLINE_NOT_IN_HEADERS=0
HOST_VISIBILITY
FMK_SUPPORT_DUMP
OS_CENTOS
)

target_compile_options(ge_common_static PRIVATE
-fvisibility=hidden
-O2
-Werror
)

target_include_directories(ge_common_static PRIVATE
${GE_CODE_DIR}/ge
${GE_CODE_DIR}/ge/common
${GE_CODE_DIR}/ge/common/op
${GE_CODE_DIR}/inc
${GE_CODE_DIR}/inc/external
${GE_CODE_DIR}/inc/framework
${METADEF_DIR}/inc
${METADEF_DIR}/inc/external
${METADEF_DIR}/inc/external/graph
${METADEF_DIR}/inc/graph
${CMAKE_BINARY_DIR}
${CMAKE_BINARY_DIR}/proto/ge
#### yellow zone ####
${GE_DEPEND_DIR}/inc
${GE_DEPEND_DIR}/inc/cce
#### blue zone ####
#${GE_DEPEND_DIR}/include
)

target_link_libraries(ge_common_static PRIVATE
$<BUILD_INTERFACE:intf_pub>
protobuf
json
c_sec
-lrt
-ldl
)

############ install ############
set(INSTALL_BASE_DIR "")
set(INSTALL_LIBRARY_DIR lib)

install(TARGETS ge_common OPTIONAL
LIBRARY DESTINATION ${INSTALL_LIBRARY_DIR}
)

+ 33
- 14
ge/common/auth/file_saver.cc View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -55,9 +55,26 @@ Status FileSaver::OpenFile(int32_t &fd, const std::string &file_path) {

Status FileSaver::WriteData(const void *data, uint32_t size, int32_t fd) {
GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(size == 0 || data == nullptr, return PARAM_INVALID);

mmSsize_t write_count;
uint32_t size_2g = ((uint32_t) 0x1 << 31);
uint32_t size_1g = ((uint32_t) 0x1 << 30);
// Write data
int32_t write_count = mmWrite(fd, const_cast<void *>(data), size);
if (size > size_2g) {
auto seek = reinterpret_cast<uint8_t *>(const_cast<void *>(data));
while (size > size_1g) {
write_count = mmWrite(fd, reinterpret_cast<void *>(seek), size_1g);
if (write_count == EN_INVALID_PARAM || write_count == EN_ERROR) {
GELOGE(FAILED, "Write data failed. mmpa_errorno = %d, %s", write_count, strerror(errno));
return FAILED;
}
size -= size_1g;
seek += size_1g;
}
write_count = mmWrite(fd, reinterpret_cast<void *>(seek), size);
} else {
write_count = mmWrite(fd, const_cast<void *>(data), size);
}

// -1: Failed to write to file; - 2: Illegal parameter
if (write_count == EN_INVALID_PARAM || write_count == EN_ERROR) {
GELOGE(FAILED, "Write data failed. mmpa_errorno = %d, %s", write_count, strerror(errno));
@@ -99,10 +116,10 @@ Status FileSaver::SaveWithFileHeader(const std::string &file_path, const ModelFi
ModelPartitionTable &model_partition_table,

const std::vector<ModelPartition> &partition_datas) {
GE_CHK_BOOL_RET_STATUS(
!partition_datas.empty() && model_partition_table.num != 0 && model_partition_table.num == partition_datas.size(),
FAILED, "Invalid param:partition data size is (%u), model_partition_table.num is (%zu).", model_partition_table.num,
partition_datas.size());
GE_CHK_BOOL_RET_STATUS(!partition_datas.empty() && model_partition_table.num != 0
&& model_partition_table.num == partition_datas.size(), FAILED,
"Invalid param:partition data size is (%u), model_partition_table.num is (%zu).",
model_partition_table.num, partition_datas.size());
// Open file
int32_t fd = 0;
GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(OpenFile(fd, file_path) != SUCCESS, return FAILED);
@@ -110,16 +127,18 @@ Status FileSaver::SaveWithFileHeader(const std::string &file_path, const ModelFi
do {
// Write file header
GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(
WriteData(static_cast<const void *>(&file_header), sizeof(ModelFileHeader), fd) != SUCCESS, ret = FAILED; break);
WriteData(static_cast<const void *>(&file_header), sizeof(ModelFileHeader), fd) != SUCCESS, ret = FAILED;
break);
// Write model partition table
uint32_t table_size = static_cast<uint32_t>(SIZE_OF_MODEL_PARTITION_TABLE(model_partition_table));
GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(
WriteData(static_cast<const void *>(&model_partition_table), table_size, fd) != SUCCESS, ret = FAILED; break);
WriteData(static_cast<const void *>(&model_partition_table), table_size, fd) != SUCCESS, ret = FAILED; break);
// Write partition data
for (const auto &partitionData : partition_datas) {
GELOGI("GC:size[%zu]", partitionData.size);
GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(
WriteData(static_cast<const void *>(partitionData.data), partitionData.size, fd) != SUCCESS, ret = FAILED;
break);
WriteData(static_cast<const void *>(partitionData.data), partitionData.size, fd) != SUCCESS, ret = FAILED;
break);
}
} while (0);
// Close file
@@ -132,9 +151,9 @@ Status FileSaver::SaveToBuffWithFileHeader(const ModelFileHeader &file_header,
const std::vector<ModelPartition> &partitionDatas,
ge::ModelBufferData &model) {
GE_CHK_BOOL_RET_STATUS(
!partitionDatas.empty() && model_partition_table.num != 0 && model_partition_table.num == partitionDatas.size(),
FAILED, "Invalid param:partition data size is (%u), model_partition_table.num is (%zu).", model_partition_table.num,
partitionDatas.size());
!partitionDatas.empty() && model_partition_table.num != 0 && model_partition_table.num == partitionDatas.size(),
FAILED, "Invalid param:partition data size is (%u), model_partition_table.num is (%zu).",
model_partition_table.num, partitionDatas.size());
uint32_t model_header_size = sizeof(ModelFileHeader);
uint32_t table_size = static_cast<uint32_t>(SIZE_OF_MODEL_PARTITION_TABLE(model_partition_table));
uint32_t total_size = model_header_size + table_size;


+ 5
- 3
ge/common/auth/file_saver.h View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -74,8 +74,10 @@ class FileSaver {
ModelPartitionTable &model_partition_table,
const std::vector<ModelPartition> &partition_datas);

static Status SaveToBuffWithFileHeader(const ModelFileHeader &file_header, ModelPartitionTable &model_partition_table,
const std::vector<ModelPartition> &partitionDatas, ge::ModelBufferData &model);
static Status SaveToBuffWithFileHeader(const ModelFileHeader &file_header,
ModelPartitionTable &model_partition_table,
const std::vector<ModelPartition> &partitionDatas,
ge::ModelBufferData& model);

static Status SaveToFile(const string &file_path, const void *data, int len);



+ 18
- 15
ge/common/base64.h View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -25,23 +25,24 @@

namespace ge {
namespace {
const char *kBase64Chars =
"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
"abcdefghijklmnopqrstuvwxyz"
"0123456789+/";
const char* kBase64Chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
"abcdefghijklmnopqrstuvwxyz"
"0123456789+/";
const char kEqualSymbol = '=';
const size_t kBase64CharsNum = 64;
const size_t kThreeByteOneGroup = 3;
const size_t kFourByteOneGroup = 4;
} // namespace
}

namespace base64 {
static inline bool IsBase64Char(const char &c) { return (isalnum(c) || (c == '+') || (c == '/')); }
static inline bool IsBase64Char(const char &c) {
return (isalnum(c) || (c == '+') || (c == '/'));
}

static std::string EncodeToBase64(const std::string &raw_data) {
size_t encode_length = raw_data.size() / kThreeByteOneGroup * kFourByteOneGroup;
encode_length += raw_data.size() % kThreeByteOneGroup == 0 ? 0 : kFourByteOneGroup;
size_t raw_data_index = 0;
size_t raw_data_index = 0 ;
size_t encode_data_index = 0;
std::string encode_data;
encode_data.resize(encode_length);
@@ -79,7 +80,8 @@ static std::string EncodeToBase64(const std::string &raw_data) {
#pragma GCC diagnostic ignored "-Wunused-function"
static Status DecodeFromBase64(const std::string &base64_data, std::string &decode_data) {
if (base64_data.size() % kFourByteOneGroup != 0) {
GELOGE(PARAM_INVALID, "base64 data size must can be divided by 4, but given data size is %zu", base64_data.size());
GELOGE(PARAM_INVALID, "base64 data size must can be divided by 4, but given data size is %zu",
base64_data.size());
return PARAM_INVALID;
}
decode_data.clear();
@@ -92,7 +94,8 @@ static Status DecodeFromBase64(const std::string &base64_data, std::string &deco

for (std::size_t input_data_index = 0; input_data_index < base64_data_len; input_data_index += 4) {
for (size_t i = 0; i < kFourByteOneGroup; ++i) {
if (base64_data[input_data_index + i] == kEqualSymbol && input_data_index >= base64_data_len - 4 && i > 1) {
if (base64_data[input_data_index + i] == kEqualSymbol &&
input_data_index >= base64_data_len - 4 && i > 1) {
byte_4[i] = kBase64CharsNum;
} else if (IsBase64Char(base64_data[input_data_index + i])) {
byte_4[i] = FindCharInBase64Chars(base64_data[input_data_index + i]);
@@ -102,18 +105,18 @@ static Status DecodeFromBase64(const std::string &base64_data, std::string &deco
}
}
decode_data += static_cast<char>((byte_4[0] << 2u) + ((byte_4[1] & 0x30) >> 4u));
if (byte_4[2] >= kBase64CharsNum) {
if (byte_4[2] >= kBase64CharsNum){
break;
} else if (byte_4[3] >= kBase64CharsNum) {
decode_data += static_cast<char>(((byte_4[1] & 0x0f) << 4u) + ((byte_4[2] & 0x3c) >> 2u));
decode_data += static_cast<char>(((byte_4[1] & 0x0f) << 4u) + ((byte_4[2] & 0x3c) >> 2u));
break;
}
decode_data += static_cast<char>(((byte_4[1] & 0x0f) << 4u) + ((byte_4[2] & 0x3c) >> 2u));
decode_data += static_cast<char>(((byte_4[2] & 0x03) << 6u) + byte_4[3]);
decode_data += static_cast<char>(((byte_4[1] & 0x0f) << 4u) + ((byte_4[2] & 0x3c) >> 2u));
decode_data += static_cast<char>(((byte_4[2] & 0x03) << 6u) + byte_4[3]);
}
return SUCCESS;
}
#pragma GCC diagnostic pop
} // namespace base64
}
} // namespace ge
#endif // GE_COMMON_BASE64_H_

+ 1
- 1
ge/common/context/ctx.cc View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.


+ 0
- 248
ge/common/convert/pb2json.cc View File

@@ -1,248 +0,0 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

// File: pb2json.h
// Description: This imply file for protobuf message and json interconversion

#include "common/convert/pb2json.h"
#include <set>
#include <string>
#include "securec.h"
#include "framework/common/fmk_types.h"
#include "framework/common/debug/ge_log.h"

using std::set;
using std::string;

namespace ge {
namespace {
const int kSignificantDigits = 10;
}
// JSON parses non utf8 character throwing exceptions, so some fields need to be shielded through black fields
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void Pb2Json::Message2Json(const ProtobufMsg &message,
const set<string> &black_fields, Json &json,
bool enum2str) {
auto descriptor = message.GetDescriptor();
auto reflection = message.GetReflection();
if (descriptor == nullptr || reflection == nullptr) {
return;
}

auto count = descriptor->field_count();

for (auto i = 0; i < count; ++i) {
const auto field = descriptor->field(i);
if (field == nullptr) {
return;
}

// Do not display weight data
if (black_fields.find(field->name()) != black_fields.end()) {
continue;
}

if (field->is_repeated()) {
if (reflection->FieldSize(message, field) > 0) {
RepeatedMessage2Json(message, field, reflection, black_fields, json[field->name()], enum2str);
}
continue;
}

if (!reflection->HasField(message, field)) {
continue;
}

OneField2Json(message, field, reflection, black_fields, json, enum2str);
}
}

void Pb2Json::OneField2Json(const ProtobufMsg &message, const ProtobufFieldDescriptor *field,
const ProtobufReflection *reflection, const set<string> &black_fields, Json &json,
bool enum2str) {
switch (field->type()) {
case ProtobufFieldDescriptor::TYPE_MESSAGE: {
const ProtobufMsg &tmp_message = reflection->GetMessage(message, field);
if (0 != tmp_message.ByteSize()) {
Message2Json(tmp_message, black_fields, json[field->name()], enum2str);
}
break;
}

case ProtobufFieldDescriptor::TYPE_BOOL:
json[field->name()] = reflection->GetBool(message, field);
break;

case ProtobufFieldDescriptor::TYPE_ENUM: {
auto *enum_value_desc = reflection->GetEnum(message, field);
Enum2Json(enum_value_desc, field, enum2str, json);
break;
}

case ProtobufFieldDescriptor::TYPE_INT32:
case ProtobufFieldDescriptor::TYPE_SINT32:
case ProtobufFieldDescriptor::TYPE_SFIXED32:
json[field->name()] = reflection->GetInt32(message, field);
break;

case ProtobufFieldDescriptor::TYPE_UINT32:
case ProtobufFieldDescriptor::TYPE_FIXED32:
json[field->name()] = reflection->GetUInt32(message, field);
break;

case ProtobufFieldDescriptor::TYPE_INT64:
case ProtobufFieldDescriptor::TYPE_SINT64:
case ProtobufFieldDescriptor::TYPE_SFIXED64:
json[field->name()] = reflection->GetInt64(message, field);
break;

case ProtobufFieldDescriptor::TYPE_UINT64:
case ProtobufFieldDescriptor::TYPE_FIXED64:
json[field->name()] = reflection->GetUInt64(message, field);
break;

case ProtobufFieldDescriptor::TYPE_FLOAT:
char str[kSignificantDigits];
if (sprintf_s(str, kSignificantDigits, "%g", reflection->GetFloat(message, field)) != -1) {
json[field->name()] = str;
} else {
json[field->name()] = reflection->GetFloat(message, field);
}

break;

case ProtobufFieldDescriptor::TYPE_STRING:
json[field->name()] = reflection->GetString(message, field);
break;

case ProtobufFieldDescriptor::TYPE_BYTES: {
string field_name = field->name();
string type_bytes = reflection->GetString(message, field);
json[field_name] = TypeBytes2String(field_name, type_bytes);
break;
}

default:
break;
}
}

string Pb2Json::TypeBytes2String(string &field_name, string &type_bytes) {
if (field_name != "offset") {
return type_bytes;
}
string result = "";
for (char temp_value : type_bytes) {
uint8_t *value = 0;
value = reinterpret_cast<uint8_t *>(&temp_value);
char str[kSignificantDigits];
if (sprintf_s(str, kSignificantDigits, "%d", *value) == -1) {
GELOGW("Convert bytes to string fail, filed name:%s", field_name.c_str());
continue;
}
result += str;
}
return result;
}

void Pb2Json::RepeatedMessage2Json(const ProtobufMsg &message, const ProtobufFieldDescriptor *field,
const ProtobufReflection *reflection, const set<string> &black_fields, Json &json,
bool enum2str) {
if ((field == nullptr) || (reflection == nullptr)) {
Message2Json(message, black_fields, json, enum2str);
return;
}

for (auto i = 0; i < reflection->FieldSize(message, field); ++i) {
Json tmp_json;
switch (field->type()) {
case ProtobufFieldDescriptor::TYPE_MESSAGE: {
const ProtobufMsg &tmp_message = reflection->GetRepeatedMessage(message, field, i);
if (0 != tmp_message.ByteSize()) {
Message2Json(tmp_message, black_fields, tmp_json, enum2str);
}
} break;

case ProtobufFieldDescriptor::TYPE_BOOL:
tmp_json = reflection->GetRepeatedBool(message, field, i);
break;

case ProtobufFieldDescriptor::TYPE_ENUM: {
auto *enum_value_desc = reflection->GetRepeatedEnum(message, field, i);
RepeatedEnum2Json(enum_value_desc, enum2str, tmp_json);
} break;

case ProtobufFieldDescriptor::TYPE_INT32:
case ProtobufFieldDescriptor::TYPE_SINT32:
case ProtobufFieldDescriptor::TYPE_SFIXED32:
tmp_json = reflection->GetRepeatedInt32(message, field, i);
break;

case ProtobufFieldDescriptor::TYPE_UINT32:
case ProtobufFieldDescriptor::TYPE_FIXED32:
tmp_json = reflection->GetRepeatedUInt32(message, field, i);
break;

case ProtobufFieldDescriptor::TYPE_INT64:
case ProtobufFieldDescriptor::TYPE_SINT64:
case ProtobufFieldDescriptor::TYPE_SFIXED64:
tmp_json = reflection->GetRepeatedInt64(message, field, i);
break;

case ProtobufFieldDescriptor::TYPE_UINT64:
case ProtobufFieldDescriptor::TYPE_FIXED64:
tmp_json = reflection->GetRepeatedUInt64(message, field, i);
break;

case ProtobufFieldDescriptor::TYPE_FLOAT:
tmp_json = reflection->GetRepeatedFloat(message, field, i);
break;

case ProtobufFieldDescriptor::TYPE_STRING:
case ProtobufFieldDescriptor::TYPE_BYTES:
tmp_json = reflection->GetRepeatedString(message, field, i);
break;

default:
break;
}
json += tmp_json;
}
}

void Pb2Json::Enum2Json(const ProtobufEnumValueDescriptor *enum_value_desc, const ProtobufFieldDescriptor *field,
bool enum2str, Json &json) {
if (enum_value_desc != nullptr) {
if (field == nullptr) {
return;
}
if (enum2str) {
json[field->name()] = enum_value_desc->name();
} else {
json[field->name()] = enum_value_desc->number();
}
}
}

void Pb2Json::RepeatedEnum2Json(const ProtobufEnumValueDescriptor *enum_value_desc, bool enum2str, Json &json) {
if (enum_value_desc != nullptr) {
if (enum2str) {
json = enum_value_desc->name();
} else {
json = enum_value_desc->number();
}
}
}
} // namespace ge

+ 0
- 68
ge/common/convert/pb2json.h View File

@@ -1,68 +0,0 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

// File: pb2json.h
// Description: This header file for protobuf message and json interconversion

#ifndef GE_COMMON_CONVERT_PB2JSON_H_
#define GE_COMMON_CONVERT_PB2JSON_H_
#include <functional>
#include <memory>
#include <set>
#include <string>
#include "google/protobuf/descriptor.h"
#include "google/protobuf/message.h"
#include "nlohmann/json.hpp"

namespace ge {
using Json = nlohmann::json;
using ProtobufMsg = ::google::protobuf::Message;
using ProtobufReflection = ::google::protobuf::Reflection;
using ProtobufFieldDescriptor = ::google::protobuf::FieldDescriptor;
using ProtobufDescriptor = ::google::protobuf::Descriptor;
using ProtobufEnumValueDescriptor = ::google::protobuf::EnumValueDescriptor;

class Pb2Json {
public:
/**
* @ingroup domi_omg
* @brief Transfer protobuf object to JSON object
* @param [out] json Converted JSON object
* @return void success
* @author
*/
static void Message2Json(const ProtobufMsg &message, const std::set<std::string> &black_fields, Json &json,
bool enum2str = false);

protected:
static void RepeatedMessage2Json(const ProtobufMsg &message, const ProtobufFieldDescriptor *field,
const ProtobufReflection *reflection, const std::set<std::string> &black_fields,
Json &json, bool enum2str);

static void Enum2Json(const ProtobufEnumValueDescriptor *enum_value_desc, const ProtobufFieldDescriptor *field,
bool enum2str, Json &json);

static void RepeatedEnum2Json(const ProtobufEnumValueDescriptor *enum_value_desc, bool enum2str, Json &json);

static void OneField2Json(const ProtobufMsg &message, const ProtobufFieldDescriptor *field,
const ProtobufReflection *reflection, const std::set<std::string> &black_fields, Json &json,
bool enum2str);

static std::string TypeBytes2String(std::string &field_name, std::string &type_bytes);
};
} // namespace ge

#endif // GE_COMMON_CONVERT_PB2JSON_H_

+ 4
- 2
ge/common/cust_aicpu_kernel_store.cc View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -20,7 +20,9 @@ namespace ge {

CustAICPUKernelStore::CustAICPUKernelStore() {}

void CustAICPUKernelStore::AddCustAICPUKernel(const CustAICPUKernelPtr &kernel) { AddKernel(kernel); }
void CustAICPUKernelStore::AddCustAICPUKernel(const CustAICPUKernelPtr &kernel) {
AddKernel(kernel);
}

void CustAICPUKernelStore::LoadCustAICPUKernelBinToOpDesc(const std::shared_ptr<ge::OpDesc> &op_desc) const {
GELOGI("LoadCustAICPUKernelBinToOpDesc in");


+ 1
- 1
ge/common/cust_aicpu_kernel_store.h View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.


+ 1
- 1
ge/common/debug/memory_dumper.cc View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.


+ 1
- 1
ge/common/debug/memory_dumper.h View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.


+ 1
- 1
ge/common/dump/dump_manager.cc View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.


+ 1
- 1
ge/common/dump/dump_manager.h View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.


+ 1
- 1
ge/common/dump/dump_op.cc View File

@@ -252,4 +252,4 @@ Status DumpOp::LaunchDumpOp() {
}
return SUCCESS;
}
} // namespace ge
} // namesapce ge

+ 1
- 1
ge/common/dump/dump_op.h View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.


+ 8
- 8
ge/common/dump/dump_properties.cc View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -35,14 +35,14 @@ const std::string kDumpStatusOpen = "on";
const uint32_t kAicoreOverflow = (0x1 << 0);
const uint32_t kAtomicOverflow = (0x1 << 1);
const uint32_t kAllOverflow = (kAicoreOverflow | kAtomicOverflow);
} // namespace
}
namespace ge {
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY DumpProperties::DumpProperties(const DumpProperties &other) {
CopyFrom(other);
}

FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY DumpProperties &DumpProperties::operator=(
const DumpProperties &other) {
const DumpProperties &other) {
CopyFrom(other);
return *this;
}
@@ -97,7 +97,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void DumpProperties::InitByOpti

// The following is the new dump scenario of the fusion operator
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void DumpProperties::AddPropertyValue(
const std::string &model, const std::set<std::string> &layers) {
const std::string &model, const std::set<std::string> &layers) {
for (const std::string &layer : layers) {
GELOGI("This model %s config to dump layer %s", model.c_str(), layer.c_str());
}
@@ -136,7 +136,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY std::set<std::string> DumpPrope
}

FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY std::set<std::string> DumpProperties::GetPropertyValue(
const std::string &model) const {
const std::string &model) const {
auto iter = model_dump_properties_map_.find(model);
if (iter != model_dump_properties_map_.end()) {
return iter->second;
@@ -145,7 +145,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY std::set<std::string> DumpPrope
}

FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY bool DumpProperties::IsLayerNeedDump(
const std::string &model, const std::string &om_name, const std::string &op_name) const {
const std::string &model, const std::string &om_name, const std::string &op_name) const {
// if dump all
if (model_dump_properties_map_.find(DUMP_ALL_MODEL) != model_dump_properties_map_.end()) {
return true;
@@ -201,7 +201,7 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY const std::string &DumpProperti
}

FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void DumpProperties::SetDumpOpSwitch(
const std::string &dump_op_switch) {
const std::string dump_op_switch) {
dump_op_switch_ = dump_op_switch;
}

@@ -266,4 +266,4 @@ void DumpProperties::SetDumpDebugOptions() {
GELOGI("ge.exec.enableDumpDebug is false or is not set.");
}
}
} // namespace ge
} // namespace

+ 6
- 6
ge/common/dump/dump_properties.h View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -65,7 +65,7 @@ class DumpProperties {

const std::string &GetDumpStatus() const;

void SetDumpOpSwitch(const std::string &dump_op_switch);
void SetDumpOpSwitch(const std::string dump_op_switch);

const std::string &GetDumpOpSwitch() const;

@@ -77,9 +77,9 @@ class DumpProperties {

uint32_t GetOpDebugMode() const { return op_debug_mode_; }

const std::string &GetEnableDump() const { return enable_dump_; }
const std::string &GetEnableDump() const {return enable_dump_;}

const std::string &GetEnableDumpDebug() const { return enable_dump_debug_; }
const std::string &GetEnableDumpDebug() const {return enable_dump_debug_;}

private:
void CopyFrom(const DumpProperties &other);
@@ -99,6 +99,6 @@ class DumpProperties {
bool is_op_debug_ = false;
uint32_t op_debug_mode_ = 0;
};
} // namespace ge
}

#endif // GE_COMMON_DUMP_DUMP_PROPERTIES_H_
#endif //GE_COMMON_DUMP_DUMP_PROPERTIES_H_

+ 21
- 0
ge/common/dump/dump_server.cc View File

@@ -0,0 +1,21 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#include "adx_datadump_server.h"

int AdxDataDumpServerUnInit() { return 0; }

int AdxDataDumpServerInit() { return 0; }

+ 1
- 1
ge/common/fmk_error_codes.cc View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.


+ 4
- 4
ge/common/formats/format_transfers/datatype_transfer.cc View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -89,8 +89,8 @@ Status TransDataSrc2Fp16(const CastArgs &args, uint8_t *dst, const size_t data_s
}

Status CastKernel(const CastArgs &args, uint8_t *dst, const size_t data_size, const DataTypeTransMode trans_mode) {
static std::map<DataTypeTransMode, std::function<Status(const CastArgs &, uint8_t *, const size_t)>> transfer_handle =
{
static std::map<DataTypeTransMode, std::function<Status(const CastArgs &, uint8_t *, const size_t)>>
transfer_handle = {
{kTransferWithDatatypeFloatToFloat16, TransDataSrc2Fp16<float>},
{kTransferWithDatatypeFloatToInt32, TransDataSrc2Dst<float, int32_t>},
{kTransferWithDatatypeFloat16ToFloat, TransDataSrc2Dst<fp16_t, float>},
@@ -107,7 +107,7 @@ Status CastKernel(const CastArgs &args, uint8_t *dst, const size_t data_size, co
{kTransferWithDatatypeInt32ToInt64, TransDataSrc2Dst<int32_t, int64_t>},
{kTransferWithDatatypeInt32ToDouble, TransDataSrc2Dst<int32_t, double>},
{kTransferWithDatatypeDoubleToInt32, TransDataSrc2Dst<double, int32_t>},
};
};
auto it = transfer_handle.find(trans_mode);
if (it == transfer_handle.end()) {
return UNSUPPORTED;


+ 1
- 1
ge/common/formats/format_transfers/datatype_transfer.h View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.


+ 3
- 3
ge/common/formats/format_transfers/format_transfer_c1hwncoc0_hwcn.cc View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -105,8 +105,8 @@ Status GetDstDataAfterTrans(const TransArgs &args, TransResult &result, int size
auto dst_offset = dst_idx * size;
// The memcpy_s/memset_s argument `dstMax` must be less than 2G
auto protected_size = total_size - dst_offset < static_cast<int64_t>(SECUREC_MEM_MAX_LEN)
? total_size - dst_offset
: static_cast<int64_t>(SECUREC_MEM_MAX_LEN);
? total_size - dst_offset
: static_cast<int64_t>(SECUREC_MEM_MAX_LEN);
auto ret = memcpy_s(dst.get() + dst_offset, static_cast<size_t>(protected_size), args.data + src_offset,
static_cast<size_t>(size));
if (ret != EOK) {


+ 1
- 1
ge/common/formats/format_transfers/format_transfer_c1hwncoc0_hwcn.h View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.


+ 1
- 2
ge/common/formats/format_transfers/format_transfer_dhwcn_fracz3D.cc View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -13,7 +13,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#include "common/formats/format_transfers/format_transfer_dhwcn_fracz3D.h"

#include <securec.h>


+ 1
- 2
ge/common/formats/format_transfers/format_transfer_dhwcn_fracz3D.h View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -13,7 +13,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#ifndef GE_COMMON_FORMATS_FORMAT_TRANSFERS_FORMAT_TRANSFER_DHWCN_FRACTAL_Z_3D_H_
#define GE_COMMON_FORMATS_FORMAT_TRANSFERS_FORMAT_TRANSFER_DHWCN_FRACTAL_Z_3D_H_



+ 1
- 2
ge/common/formats/format_transfers/format_transfer_dhwnc_fracz3D_transpose.cc View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -13,7 +13,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#include "common/formats/format_transfers/format_transfer_dhwnc_fracz3D_transpose.h"

#include <securec.h>


+ 1
- 2
ge/common/formats/format_transfers/format_transfer_dhwnc_fracz3D_transpose.h View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -13,7 +13,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#ifndef GE_COMMON_FORMATS_FORMAT_TRANSFERS_FORMAT_TRANSFER_DHWNC_FRACTAL_Z_3D_TRANSPOSE_H_
#define GE_COMMON_FORMATS_FORMAT_TRANSFERS_FORMAT_TRANSFER_DHWNC_FRACTAL_Z_3D_TRANSPOSE_H_



+ 9
- 13
ge/common/formats/format_transfers/format_transfer_fractal_nz.cc View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -154,9 +154,8 @@ Status TransFormatFromNdToFracNz(const TransArgs &args, TransResult &result, con
for (int64_t w1_idx = 0; w1_idx < num_w1; w1_idx++) {
auto dst_offset = (h1h0_head + w1_idx * h1h0w0) * size;
auto src_offset = (src_h_head + w1_idx * w0) * size;
auto protected_size = dst_size - dst_offset < static_cast<int64_t>(SECUREC_MEM_MAX_LEN)
? dst_size - dst_offset
: static_cast<int64_t>(SECUREC_MEM_MAX_LEN);
auto protected_size = dst_size - dst_offset < static_cast<int64_t>(SECUREC_MEM_MAX_LEN) ?
dst_size - dst_offset : static_cast<int64_t>(SECUREC_MEM_MAX_LEN);
auto ret = memcpy_s(dst.get() + dst_offset, static_cast<size_t>(protected_size), args.data + src_offset,
static_cast<size_t>(size * w0));
if (ret != EOK) {
@@ -169,9 +168,8 @@ Status TransFormatFromNdToFracNz(const TransArgs &args, TransResult &result, con
auto src_w_idx = w1_head + w0_idx;
auto dst_offset = (h1h0_head + num_w1 * h1h0w0 + w0_idx) * size;
auto src_offset = (src_h_head + src_w_idx) * size;
auto protected_size = dst_size - dst_offset < static_cast<int64_t>(SECUREC_MEM_MAX_LEN)
? dst_size - dst_offset
: static_cast<int64_t>(SECUREC_MEM_MAX_LEN);
auto protected_size = dst_size - dst_offset < static_cast<int64_t>(SECUREC_MEM_MAX_LEN) ?
dst_size - dst_offset : static_cast<int64_t>(SECUREC_MEM_MAX_LEN);
auto ret = memcpy_s(dst.get() + dst_offset, static_cast<size_t>(protected_size), args.data + src_offset,
static_cast<size_t>(size));
if (ret != EOK) {
@@ -227,9 +225,8 @@ Status TransFormatFromFracNzToNd(const TransArgs &args, TransResult &result, con
for (int64_t w1_idx = 0; w1_idx < num_w1; w1_idx++) {
auto src_offset = (h1h0_head + w1_idx * h1h0w0) * size;
auto dst_offset = (dst_h_head + w1_idx * w0) * size;
auto protected_size = dst_size - dst_offset < static_cast<int64_t>(SECUREC_MEM_MAX_LEN)
? dst_size - dst_offset
: static_cast<int64_t>(SECUREC_MEM_MAX_LEN);
auto protected_size = dst_size - dst_offset < static_cast<int64_t>(SECUREC_MEM_MAX_LEN) ?
dst_size - dst_offset : static_cast<int64_t>(SECUREC_MEM_MAX_LEN);
ret = memcpy_s(dst.get() + dst_offset, static_cast<size_t>(protected_size), args.data + src_offset,
static_cast<size_t>(size * w0));
if (ret != EOK) {
@@ -242,9 +239,8 @@ Status TransFormatFromFracNzToNd(const TransArgs &args, TransResult &result, con
auto dst_w_idx = w1_head + w0_idx;
auto src_offset = (h1h0_head + num_w1 * h1h0w0 + w0_idx) * size;
auto dst_offset = (dst_h_head + dst_w_idx) * size;
auto protected_size = dst_size - dst_offset < static_cast<int64_t>(SECUREC_MEM_MAX_LEN)
? dst_size - dst_offset
: static_cast<int64_t>(SECUREC_MEM_MAX_LEN);
auto protected_size = dst_size - dst_offset < static_cast<int64_t>(SECUREC_MEM_MAX_LEN) ?
dst_size - dst_offset : static_cast<int64_t>(SECUREC_MEM_MAX_LEN);
ret = memcpy_s(dst.get() + dst_offset, static_cast<size_t>(protected_size), args.data + src_offset,
static_cast<size_t>(size));
if (ret != EOK) {


+ 1
- 1
ge/common/formats/format_transfers/format_transfer_fractal_nz.h View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.


+ 22
- 22
ge/common/formats/format_transfers/format_transfer_fractal_z.cc View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -124,11 +124,11 @@ Status TransFormatFromNchwToFz(const TransArgs &args, TransResult &result) {

std::shared_ptr<uint8_t> dst(new (std::nothrow) uint8_t[dst_size], std::default_delete<uint8_t[]>());
GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(
dst == nullptr,
GELOGE(OUT_OF_MEMORY, "Failed to trans format from %s to %s, can not alloc the memory for dst buf %ld",
TypeUtils::FormatToSerialString(args.src_format).c_str(),
TypeUtils::FormatToSerialString(args.dst_format).c_str(), dst_size);
return OUT_OF_MEMORY;);
dst == nullptr,
GELOGE(OUT_OF_MEMORY, "Failed to trans format from %s to %s, can not alloc the memory for dst buf %ld",
TypeUtils::FormatToSerialString(args.src_format).c_str(),
TypeUtils::FormatToSerialString(args.dst_format).c_str(), dst_size);
return OUT_OF_MEMORY;);

for (int64_t vfi = 0; vfi < vf_cnt; vfi++) {
// vertical fractal matrix base index
@@ -152,8 +152,8 @@ Status TransFormatFromNchwToFz(const TransArgs &args, TransResult &result) {
auto idx = gfi * fractal_ele_cnt + col * c0 + row;
auto offset = idx * size;
auto protected_size = dst_size - offset < static_cast<int64_t>(SECUREC_MEM_MAX_LEN)
? dst_size - offset
: static_cast<int64_t>(SECUREC_MEM_MAX_LEN);
? dst_size - offset
: static_cast<int64_t>(SECUREC_MEM_MAX_LEN);
errno_t ret = EOK;
if (need_pad_zero) {
ret = memset_s(dst.get() + offset, static_cast<size_t>(protected_size), 0, static_cast<size_t>(size));
@@ -209,11 +209,11 @@ Status TransFormatHwcnToFz(const TransArgs &args, TransResult &result) {

std::shared_ptr<uint8_t> dst(new (std::nothrow) uint8_t[dst_size], std::default_delete<uint8_t[]>());
GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(
dst == nullptr,
GELOGE(OUT_OF_MEMORY, "Failed to trans format from %s to %s, can not alloc the memory for dst buf %ld",
TypeUtils::FormatToSerialString(args.src_format).c_str(),
TypeUtils::FormatToSerialString(args.dst_format).c_str(), dst_size);
return OUT_OF_MEMORY;);
dst == nullptr,
GELOGE(OUT_OF_MEMORY, "Failed to trans format from %s to %s, can not alloc the memory for dst buf %ld",
TypeUtils::FormatToSerialString(args.src_format).c_str(),
TypeUtils::FormatToSerialString(args.dst_format).c_str(), dst_size);
return OUT_OF_MEMORY;);

for (int64_t c1i = 0; c1i < c1; c1i++) {
for (int64_t hi = 0; hi < h; hi++) {
@@ -223,8 +223,8 @@ Status TransFormatHwcnToFz(const TransArgs &args, TransResult &result) {
int64_t dst_idx = c1i * hwn1n0c0 + hi * wn1n0c0 + wi * n1n0c0 + n1n0i * c0 + c0i;
int64_t dst_offset = dst_idx * data_size;
auto protected_size = dst_size - dst_offset < static_cast<int64_t>(SECUREC_MEM_MAX_LEN)
? dst_size - dst_offset
: static_cast<int64_t>(SECUREC_MEM_MAX_LEN);
? dst_size - dst_offset
: static_cast<int64_t>(SECUREC_MEM_MAX_LEN);
auto pad_zero = ((c1i * c0 + c0i) >= c) || (n1n0i >= n);
errno_t ret = EOK;
if (pad_zero) {
@@ -284,11 +284,11 @@ Status TransFormatNhwcToFz(const TransArgs &args, TransResult &result) {

std::shared_ptr<uint8_t> dst(new (std::nothrow) uint8_t[dst_size], std::default_delete<uint8_t[]>());
GE_CHK_BOOL_TRUE_EXEC_WITH_LOG(
dst == nullptr,
GELOGE(OUT_OF_MEMORY, "Failed to trans format from %s to %s, can not alloc the memory for dst buf %ld",
TypeUtils::FormatToSerialString(args.src_format).c_str(),
TypeUtils::FormatToSerialString(args.dst_format).c_str(), dst_size);
return OUT_OF_MEMORY;);
dst == nullptr,
GELOGE(OUT_OF_MEMORY, "Failed to trans format from %s to %s, can not alloc the memory for dst buf %ld",
TypeUtils::FormatToSerialString(args.src_format).c_str(),
TypeUtils::FormatToSerialString(args.dst_format).c_str(), dst_size);
return OUT_OF_MEMORY;);

for (int64_t c1i = 0; c1i < c1; c1i++) {
for (int64_t hi = 0; hi < h; hi++) {
@@ -298,8 +298,8 @@ Status TransFormatNhwcToFz(const TransArgs &args, TransResult &result) {
int64_t dst_idx = c1i * hwn1n0c0 + hi * wn1n0c0 + wi * n1n0c0 + n1n0i * c0 + c0i;
int64_t dst_offset = dst_idx * data_size;
auto protected_size = dst_size - dst_offset < static_cast<int64_t>(SECUREC_MEM_MAX_LEN)
? dst_size - dst_offset
: static_cast<int64_t>(SECUREC_MEM_MAX_LEN);
? dst_size - dst_offset
: static_cast<int64_t>(SECUREC_MEM_MAX_LEN);
auto pad_zero = ((c1i * c0 + c0i) >= c) || (n1n0i >= n);
errno_t ret = EOK;
if (pad_zero) {


+ 1
- 1
ge/common/formats/format_transfers/format_transfer_fractal_z.h View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.


+ 9
- 9
ge/common/formats/format_transfers/format_transfer_fractal_zz.cc View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2019 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -158,8 +158,8 @@ Status TransFormatFromNdToFracZz(const TransArgs &args, TransResult &result, con
auto src_offset = (src_h_head + w1_idx * w0) * size;
auto dst_offset = (h0_head + w1_idx * h0w0) * size;
auto protected_size = dst_size - dst_offset < static_cast<int64_t>(SECUREC_MEM_MAX_LEN)
? dst_size - dst_offset
: static_cast<int64_t>(SECUREC_MEM_MAX_LEN);
? dst_size - dst_offset
: static_cast<int64_t>(SECUREC_MEM_MAX_LEN);
auto ret = memcpy_s(dst.get() + dst_offset, static_cast<size_t>(protected_size), args.data + src_offset,
static_cast<size_t>(size * w0));
if (ret != EOK) {
@@ -174,8 +174,8 @@ Status TransFormatFromNdToFracZz(const TransArgs &args, TransResult &result, con
auto src_offset = (src_h_head + src_w_idx) * size;
auto dst_offset = (w0_head + w0_idx) * size;
auto protected_size = dst_size - dst_offset < static_cast<int64_t>(SECUREC_MEM_MAX_LEN)
? dst_size - dst_offset
: static_cast<int64_t>(SECUREC_MEM_MAX_LEN);
? dst_size - dst_offset
: static_cast<int64_t>(SECUREC_MEM_MAX_LEN);
auto ret = memcpy_s(dst.get() + dst_offset, static_cast<size_t>(protected_size), args.data + src_offset,
static_cast<size_t>(size));
if (ret != EOK) {
@@ -236,8 +236,8 @@ Status TransFormatFromFracZzToNd(const TransArgs &args, TransResult &result, con
auto src_offset = (h0_head + w1_idx * h0w0) * size;
auto dst_offset = (dst_h_head + w1_idx * w0) * size;
auto protected_size = dst_size - dst_offset < static_cast<int64_t>(SECUREC_MEM_MAX_LEN)
? dst_size - dst_offset
: static_cast<int64_t>(SECUREC_MEM_MAX_LEN);
? dst_size - dst_offset
: static_cast<int64_t>(SECUREC_MEM_MAX_LEN);
auto ret = memcpy_s(dst.get() + dst_offset, static_cast<size_t>(protected_size), args.data + src_offset,
static_cast<size_t>(size * w0));
if (ret != EOK) {
@@ -252,8 +252,8 @@ Status TransFormatFromFracZzToNd(const TransArgs &args, TransResult &result, con
auto dst_w_idx = w1_head + w0_idx;
auto dst_offset = (dst_h_head + dst_w_idx) * size;
auto protected_size = dst_size - dst_offset < static_cast<int64_t>(SECUREC_MEM_MAX_LEN)
? dst_size - dst_offset
: static_cast<int64_t>(SECUREC_MEM_MAX_LEN);
? dst_size - dst_offset
: static_cast<int64_t>(SECUREC_MEM_MAX_LEN);
auto ret = memcpy_s(dst.get() + dst_offset, static_cast<size_t>(protected_size), args.data + src_offset,
static_cast<size_t>(size));
if (ret != EOK) {


+ 1
- 1
ge/common/formats/format_transfers/format_transfer_fractal_zz.h View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2019 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.


+ 4
- 5
ge/common/formats/format_transfers/format_transfer_fracz_hwcn.cc View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -104,9 +104,8 @@ Status GetDstDataAfterTrans(const TransArgs &args, TransResult &result, const in
int64_t src_idx = c1_idx * hwncc0 + h_idx * wncc0 + w_idx * ncc0 + nc_idx * c0 + c0_idx;
auto src_offset = src_idx * size;
auto dst_offset = dst_idx * size;
auto protected_size = total_size - dst_offset < static_cast<int64_t>(SECUREC_MEM_MAX_LEN)
? total_size - dst_offset
: static_cast<int64_t>(SECUREC_MEM_MAX_LEN);
auto protected_size = total_size - dst_offset < static_cast<int64_t>(SECUREC_MEM_MAX_LEN) ?
total_size - dst_offset : static_cast<int64_t>(SECUREC_MEM_MAX_LEN);
auto ret = memcpy_s(dst.get() + dst_offset, static_cast<size_t>(protected_size), args.data + src_offset,
static_cast<size_t>(size));
if (ret != EOK) {
@@ -140,7 +139,7 @@ Status FormatTransferFracZHwcn::TransFormat(const TransArgs &args, TransResult &
}

GELOGE(INTERNAL_ERROR, "Get %ld total size from dst shape %s, src shape %s", total_size,
ShapeToString(args.dst_shape).c_str(), ShapeToString(args.src_shape).c_str());
ShapeToString(args.dst_shape).c_str(), ShapeToString(args.src_shape).c_str());
return PARAM_INVALID;
}
GELOGD("Begin to trans format from FracZ to HWCN, src shape %s, data type %s, dst shape %s, memory size %ld",


+ 1
- 1
ge/common/formats/format_transfers/format_transfer_fracz_hwcn.h View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.


+ 4
- 5
ge/common/formats/format_transfers/format_transfer_fracz_nchw.cc View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -104,9 +104,8 @@ Status GetDstDataAfterTrans(const TransArgs &args, TransResult &result, const in
int64_t src_idx = c1_idx * hwncc0 + h_idx * wncc0 + w_idx * ncc0 + nc_idx * c0 + c0_idx;
auto src_offset = src_idx * size;
auto dst_offset = dst_idx * size;
auto protected_size = total_size - dst_offset < static_cast<int64_t>(SECUREC_MEM_MAX_LEN)
? total_size - dst_offset
: static_cast<int64_t>(SECUREC_MEM_MAX_LEN);
auto protected_size = total_size - dst_offset < static_cast<int64_t>(SECUREC_MEM_MAX_LEN) ?
total_size - dst_offset : static_cast<int64_t>(SECUREC_MEM_MAX_LEN);
auto ret = memcpy_s(dst.get() + dst_offset, static_cast<size_t>(protected_size), args.data + src_offset,
static_cast<size_t>(size));
if (ret != EOK) {
@@ -140,7 +139,7 @@ Status FormatTransferFracZNchw::TransFormat(const TransArgs &args, TransResult &
}

GELOGE(INTERNAL_ERROR, "Get %ld total size from dst shape %s, src shape %s", total_size,
ShapeToString(args.dst_shape).c_str(), ShapeToString(args.src_shape).c_str());
ShapeToString(args.dst_shape).c_str(), ShapeToString(args.src_shape).c_str());
return PARAM_INVALID;
}
GELOGD("Begin to trans format from FracZ to NCHW, src shape %s, data type %s, dst shape %s, memory size %ld",


+ 1
- 1
ge/common/formats/format_transfers/format_transfer_fracz_nchw.h View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.


+ 4
- 5
ge/common/formats/format_transfers/format_transfer_fracz_nhwc.cc View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -104,9 +104,8 @@ Status GetDstDataAfterTrans(const TransArgs &args, TransResult &result, int size
int64_t src_idx = c1_idx * hwncc0 + h_idx * wncc0 + w_idx * ncc0 + nc_idx * c0 + c0_idx;
auto src_offset = src_idx * size;
auto dst_offset = dst_idx * size;
auto protected_size = total_size - dst_offset < static_cast<int64_t>(SECUREC_MEM_MAX_LEN)
? total_size - dst_offset
: static_cast<int64_t>(SECUREC_MEM_MAX_LEN);
auto protected_size = total_size - dst_offset < static_cast<int64_t>(SECUREC_MEM_MAX_LEN) ?
total_size - dst_offset : static_cast<int64_t>(SECUREC_MEM_MAX_LEN);
auto ret = memcpy_s(dst.get() + dst_offset, static_cast<size_t>(protected_size), args.data + src_offset,
static_cast<size_t>(size));
if (ret != EOK) {
@@ -139,7 +138,7 @@ Status FormatTransferFracZNhwc::TransFormat(const TransArgs &args, TransResult &
}

GELOGE(INTERNAL_ERROR, "Get %ld total size from dst shape %s, src shape %s", total_size,
ShapeToString(args.dst_shape).c_str(), ShapeToString(args.src_shape).c_str());
ShapeToString(args.dst_shape).c_str(), ShapeToString(args.src_shape).c_str());
return PARAM_INVALID;
}
GELOGD("Begin to trans format from FracZ to NHWC, src shape %s, data type %s, dst shape %s, memory size %ld",


+ 1
- 1
ge/common/formats/format_transfers/format_transfer_fracz_nhwc.h View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.


+ 4
- 4
ge/common/formats/format_transfers/format_transfer_hwcn_c1hwncoc0.cc View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -122,8 +122,8 @@ Status GetDstDataAfterTrans(const TransArgs &args, TransResult &result, const in
int64_t dst_idx = c0_idx + co_head_addr;
auto dst_offset = dst_idx * size;
auto protected_size = total_size - dst_offset < static_cast<int64_t>(SECUREC_MEM_MAX_LEN)
? total_size - dst_offset
: static_cast<int64_t>(SECUREC_MEM_MAX_LEN);
? total_size - dst_offset
: static_cast<int64_t>(SECUREC_MEM_MAX_LEN);
int64_t c_idx = c0_idx + c1_idx * c0;
int64_t src_idx = h_idx * wcn + w_idx * cn + c_idx * n + n_idx;
auto src_offset = src_idx * size;
@@ -141,7 +141,7 @@ Status GetDstDataAfterTrans(const TransArgs &args, TransResult &result, const in
}
} else {
auto ret =
memset_s(dst.get() + dst_offset, static_cast<size_t>(protected_size), 0, static_cast<size_t>(size));
memset_s(dst.get() + dst_offset, static_cast<size_t>(protected_size), 0, static_cast<size_t>(size));
if (ret != EOK) {
GELOGE(INTERNAL_ERROR,
"Failed to set to 0 to C1HWNCoC0[%ld, %ld, %ld, %ld, %ld, %ld] offset %ld, "


+ 1
- 1
ge/common/formats/format_transfers/format_transfer_hwcn_c1hwncoc0.h View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.


+ 3
- 3
ge/common/formats/format_transfers/format_transfer_nc1hwc0_nchw.cc View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -102,8 +102,8 @@ Status GetDstDataAfterTrans(const TransArgs &args, TransResult &result, const in
auto src_offset = src_idx * size;
auto dst_offset = dst_idx * size;
auto protected_size = total_size - dst_offset < static_cast<int64_t>(SECUREC_MEM_MAX_LEN)
? total_size - dst_offset
: static_cast<int64_t>(SECUREC_MEM_MAX_LEN);
? total_size - dst_offset
: static_cast<int64_t>(SECUREC_MEM_MAX_LEN);
auto ret = memcpy_s(dst.get() + dst_offset, static_cast<size_t>(protected_size), args.data + src_offset,
static_cast<size_t>(size));
if (ret != EOK) {


+ 1
- 1
ge/common/formats/format_transfers/format_transfer_nc1hwc0_nchw.h View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.


+ 3
- 3
ge/common/formats/format_transfers/format_transfer_nc1hwc0_nhwc.cc View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -102,8 +102,8 @@ Status GetDstDataAfterTrans(const TransArgs &args, TransResult &result, const in
auto src_offset = src_idx * size;
auto dst_offset = dst_idx * size;
auto protected_size = total_size - dst_offset < static_cast<int64_t>(SECUREC_MEM_MAX_LEN)
? total_size - dst_offset
: static_cast<int64_t>(SECUREC_MEM_MAX_LEN);
? total_size - dst_offset
: static_cast<int64_t>(SECUREC_MEM_MAX_LEN);
auto ret = memcpy_s(dst.get() + dst_offset, static_cast<size_t>(protected_size), args.data + src_offset,
static_cast<size_t>(size));
if (ret != EOK) {


+ 1
- 1
ge/common/formats/format_transfers/format_transfer_nc1hwc0_nhwc.h View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.


+ 1
- 1
ge/common/formats/format_transfers/format_transfer_nchw_fz_c04.cc View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.


+ 1
- 1
ge/common/formats/format_transfers/format_transfer_nchw_fz_c04.h View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.


+ 8
- 8
ge/common/formats/format_transfers/format_transfer_nchw_nc1hwc0.cc View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -115,8 +115,8 @@ Status GetDstDataAfterTrans(const TransArgs &args, TransResult &result, const in
int64_t dst_index = c0_idx + w_head_addr;
int64_t dst_offset = dst_index * size;
auto protected_size = total_size - dst_offset < static_cast<int64_t>(SECUREC_MEM_MAX_LEN)
? total_size - dst_offset
: static_cast<int64_t>(SECUREC_MEM_MAX_LEN);
? total_size - dst_offset
: static_cast<int64_t>(SECUREC_MEM_MAX_LEN);
int64_t cIdx = c0_idx + c1_idx * c0;
int64_t srcIdx = n_idx * chw + cIdx * hw + h_idx * w + w_idx;
auto src_offset = srcIdx * size;
@@ -133,7 +133,7 @@ Status GetDstDataAfterTrans(const TransArgs &args, TransResult &result, const in
}
} else {
auto ret =
memset_s(dst.get() + dst_offset, static_cast<size_t>(protected_size), 0, static_cast<size_t>(size));
memset_s(dst.get() + dst_offset, static_cast<size_t>(protected_size), 0, static_cast<size_t>(size));
if (ret != EOK) {
GELOGE(INTERNAL_ERROR,
"Failed to set to 0 to "
@@ -173,10 +173,10 @@ Status FormatTransferNchwNc1hwc0::TransFormat(const TransArgs &args, TransResult
return PARAM_INVALID;
}
GELOGD(
"Begin to trans format from NCHW to NC1HWC0, src shape %s, data type "
"%s, dst shape %s memory size %ld",
ShapeToString(args.src_shape).c_str(), TypeUtils::DataTypeToSerialString(args.src_data_type).c_str(),
ShapeToString(args.dst_shape).c_str(), total_size);
"Begin to trans format from NCHW to NC1HWC0, src shape %s, data type "
"%s, dst shape %s memory size %ld",
ShapeToString(args.src_shape).c_str(), TypeUtils::DataTypeToSerialString(args.src_data_type).c_str(),
ShapeToString(args.dst_shape).c_str(), total_size);
if (GetDstDataAfterTrans(args, result, size, total_size) != SUCCESS) {
GELOGE(INTERNAL_ERROR, "Failed to get data after trans, src shape %s, data type %s, dst shape %s, memory size %ld",
ShapeToString(args.src_shape).c_str(), TypeUtils::DataTypeToSerialString(args.src_data_type).c_str(),


+ 1
- 1
ge/common/formats/format_transfers/format_transfer_nchw_nc1hwc0.h View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.


+ 3
- 3
ge/common/formats/format_transfers/format_transfer_nhwc_nc1hwc0.cc View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -119,8 +119,8 @@ Status GetDstDataAfterTrans(const TransArgs &args, TransResult &result, const in
int64_t dst_idx = c0_idx + w_head_addr;
int64_t dst_offset = dst_idx * size;
auto protected_size = total_size - dst_offset < static_cast<int64_t>(SECUREC_MEM_MAX_LEN)
? total_size - dst_offset
: static_cast<int64_t>(SECUREC_MEM_MAX_LEN);
? total_size - dst_offset
: static_cast<int64_t>(SECUREC_MEM_MAX_LEN);
int64_t c_idx = c0_idx + c1_idx * c0;
int64_t src_idx = n_idx * hwc + h_idx * wc + w_idx * c + c_idx;
auto src_offset = src_idx * size;


+ 1
- 1
ge/common/formats/format_transfers/format_transfer_nhwc_nc1hwc0.h View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.


+ 19
- 19
ge/common/formats/format_transfers/format_transfer_transpose.cc View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -28,22 +28,22 @@ namespace ge {
namespace formats {
namespace {
std::map<Format, std::map<Format, std::vector<int64_t>>> perm_args{
{FORMAT_NCHW,
{{FORMAT_NHWC, std::vector<int64_t>({0, 2, 3, 1})},
{FORMAT_HWCN, std::vector<int64_t>({2, 3, 1, 0})},
{FORMAT_CHWN, std::vector<int64_t>({1, 2, 3, 0})}}},
{FORMAT_NHWC,
{{FORMAT_NCHW, std::vector<int64_t>({0, 3, 1, 2})},
{FORMAT_CHWN, std::vector<int64_t>({3, 1, 2, 0})},
{FORMAT_HWCN, std::vector<int64_t>({1, 2, 3, 0})}}},
{FORMAT_HWCN,
{{FORMAT_NCHW, std::vector<int64_t>({3, 2, 0, 1})},
{FORMAT_NHWC, std::vector<int64_t>({3, 0, 1, 2})},
{FORMAT_CHWN, std::vector<int64_t>({2, 0, 1, 3})}}},
{FORMAT_CHWN,
{{FORMAT_NCHW, std::vector<int64_t>({3, 0, 1, 2})},
{FORMAT_NHWC, std::vector<int64_t>({3, 1, 2, 0})},
{FORMAT_HWCN, std::vector<int64_t>({1, 2, 0, 3})}}},
{FORMAT_NCHW,
{{FORMAT_NHWC, std::vector<int64_t>({0, 2, 3, 1})},
{FORMAT_HWCN, std::vector<int64_t>({2, 3, 1, 0})},
{FORMAT_CHWN, std::vector<int64_t>({1, 2, 3, 0})}}},
{FORMAT_NHWC,
{{FORMAT_NCHW, std::vector<int64_t>({0, 3, 1, 2})},
{FORMAT_CHWN, std::vector<int64_t>({3, 1, 2, 0})},
{FORMAT_HWCN, std::vector<int64_t>({1, 2, 3, 0})}}},
{FORMAT_HWCN,
{{FORMAT_NCHW, std::vector<int64_t>({3, 2, 0, 1})},
{FORMAT_NHWC, std::vector<int64_t>({3, 0, 1, 2})},
{FORMAT_CHWN, std::vector<int64_t>({2, 0, 1, 3})}}},
{FORMAT_CHWN,
{{FORMAT_NCHW, std::vector<int64_t>({3, 0, 1, 2})},
{FORMAT_NHWC, std::vector<int64_t>({3, 1, 2, 0})},
{FORMAT_HWCN, std::vector<int64_t>({1, 2, 0, 3})}}},
};

bool IsShapeArgValid(const std::vector<int64_t> &src_shape, const std::vector<int64_t> &perm_arg) {
@@ -163,8 +163,8 @@ Status Transpose(const uint8_t *src, const std::vector<int64_t> &src_shape, Data
auto src_offset = GenOffset(src_heads, dst_indexes) * data_size;
auto dst_offset_bytes = dst_index * data_size;
auto protected_size = dst_size - dst_offset_bytes < static_cast<int64_t>(SECUREC_MEM_MAX_LEN)
? dst_size - dst_offset_bytes
: static_cast<int64_t>(SECUREC_MEM_MAX_LEN);
? dst_size - dst_offset_bytes
: static_cast<int64_t>(SECUREC_MEM_MAX_LEN);
auto ret = memcpy_s(dst.get() + dst_offset_bytes, static_cast<size_t>(protected_size), src + src_offset,
static_cast<size_t>(data_size));
if (ret != EOK) {


+ 2
- 1
ge/common/formats/format_transfers/format_transfer_transpose.h View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -33,6 +33,7 @@ Status TransposeWithShapeCheck(const uint8_t *src, const std::vector<int64_t> &s

Status GetPermByForamt(Format src_format, Format dst_format, std::vector<int64_t> &perm);


class FormatTransferTranspose : public FormatTransfer {
public:
Status TransFormat(const TransArgs &args, TransResult &result) override;


+ 3
- 2
ge/common/formats/formats.cc View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2019 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -51,7 +51,8 @@ GE_FUNC_DEV_VISIBILITY GE_FUNC_HOST_VISIBILITY Status TransFormat(const TransArg

GE_FUNC_DEV_VISIBILITY GE_FUNC_HOST_VISIBILITY Status TransShape(Format src_format,
const std::vector<int64_t> &src_shape,
DataType data_type, Format dst_format,
DataType data_type,
Format dst_format,
std::vector<int64_t> &dst_shape) {
formats::TransArgs args;
args.src_format = src_format;


+ 3
- 3
ge/common/formats/formats.h View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -36,8 +36,8 @@ namespace formats {
*/
Status TransFormat(const TransArgs &args, TransResult &result);

Status TransShape(Format src_format, const std::vector<int64_t> &src_shape, DataType data_type, Format dst_format,
std::vector<int64_t> &dst_shape);
Status TransShape(Format src_format, const std::vector<int64_t> &src_shape, DataType data_type,
Format dst_format, std::vector<int64_t> &dst_shape);

Status TransDataType(const CastArgs &args, TransResult &result);



+ 53
- 8
ge/common/formats/utils/formats_definitions.h View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2019 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -23,13 +23,38 @@ static const int kCubeSize = 16;
static const int kNiSize = 16;
static const int64_t kShapeItemNumMAX = 1024UL * 1024UL * 1024UL * 1024UL;

enum NchwDimIndex { kNchwN, kNchwC, kNchwH, kNchwW, kNchwDimsNum };
enum NchwDimIndex {
kNchwN,
kNchwC,
kNchwH,
kNchwW,
kNchwDimsNum
};

enum NhwcDimIndex { kNhwcN, kNhwcH, kNhwcW, kNhwcC, kNhwcDimsNum };
enum NhwcDimIndex {
kNhwcN,
kNhwcH,
kNhwcW,
kNhwcC,
kNhwcDimsNum
};

enum HwcnDimIndex { kHwcnH, kHwcnW, kHwcnC, kHwcnN, kHwcnDimsNum };
enum HwcnDimIndex {
kHwcnH,
kHwcnW,
kHwcnC,
kHwcnN,
kHwcnDimsNum
};

enum Nc1hwc0DimIndex { kNc1hwc0N, kNc1hwc0C1, kNc1hwc0H, kNc1hwc0W, kNc1hwc0C0, kNc1hwc0DimsNum };
enum Nc1hwc0DimIndex {
kNc1hwc0N,
kNc1hwc0C1,
kNc1hwc0H,
kNc1hwc0W,
kNc1hwc0C0,
kNc1hwc0DimsNum
};

enum C1hwncoc0DimIndex {
kC1hwncoc0C1,
@@ -41,11 +66,31 @@ enum C1hwncoc0DimIndex {
kC1hwncoc0DimsNum
};

enum FracZDimIndex { kFracZHWC1, kFracZN0, kFracZNi, kFracZC0, kFracZDimsNum };
enum FracZDimIndex {
kFracZHWC1,
kFracZN0,
kFracZNi,
kFracZC0,
kFracZDimsNum
};

enum DhwcnDimIndex { kDhwcnD, kDhwcnH, kDhwcnW, kDhwcnC, kDhwcnN, kDhwcnDimsNum };
enum DhwcnDimIndex {
kDhwcnD,
kDhwcnH,
kDhwcnW,
kDhwcnC,
kDhwcnN,
kDhwcnDimsNum
};

enum DhwncDimIndex { kDhwncD, kDhwncH, kDhwncW, kDhwncN, kDhwncC, kDhwncDimsNum };
enum DhwncDimIndex {
kDhwncD,
kDhwncH,
kDhwncW,
kDhwncN,
kDhwncC,
kDhwncDimsNum
};
} // namespace formats
} // namespace ge
#endif // GE_COMMON_FORMATS_UTILS_FORMATS_DEFINITIONS_H_

+ 1
- 1
ge/common/formats/utils/formats_trans_utils.cc View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.


+ 1
- 1
ge/common/formats/utils/formats_trans_utils.h View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2019 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.


+ 1
- 1
ge/common/fp16_t.cc View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.


+ 1
- 1
ge/common/fp16_t.h View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.


+ 37
- 37
ge/common/ge/datatype_util.cc View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -22,46 +22,46 @@
namespace {
const std::vector<ge::DataType> kEmptyDatatypeVector;
std::map<ge::DataType, std::vector<ge::DataType>> g_translatable_data_type = {
// key:src datatype, value:dst datatype
{ge::DT_FLOAT, {ge::DT_FLOAT16, ge::DT_FLOAT}},
{ge::DT_BOOL, {ge::DT_INT32}},
{ge::DT_FLOAT16, {ge::DT_FLOAT, ge::DT_FLOAT16}},
{ge::DT_INT64, {ge::DT_INT32}}};
// key:src datatype, value:dst datatype
{ge::DT_FLOAT, {ge::DT_FLOAT16, ge::DT_FLOAT}},
{ge::DT_BOOL, {ge::DT_INT32}},
{ge::DT_FLOAT16, {ge::DT_FLOAT, ge::DT_FLOAT16}},
{ge::DT_INT64, {ge::DT_INT32}}};

std::map<ge::DataType, std::vector<ge::DataType>> g_reverse_translatable_data_type = {
// key:dst datatype,value:src datatype
{ge::DT_FLOAT16, {ge::DT_FLOAT, ge::DT_FLOAT16}},
{ge::DT_INT32, {ge::DT_BOOL, ge::DT_INT64}},
{ge::DT_FLOAT, {ge::DT_FLOAT16, ge::DT_FLOAT}}};
// key:dst datatype,value:src datatype
{ge::DT_FLOAT16, {ge::DT_FLOAT, ge::DT_FLOAT16}},
{ge::DT_INT32, {ge::DT_BOOL, ge::DT_INT64}},
{ge::DT_FLOAT, {ge::DT_FLOAT16, ge::DT_FLOAT}}};

static const std::map<ge::DataType, ge::proto::DataType> g_dump_data_type_map = {
// key:ge datatype,value:proto datatype
{ge::DT_UNDEFINED, ge::proto::DT_UNDEFINED},
{ge::DT_FLOAT, ge::proto::DT_FLOAT},
{ge::DT_FLOAT16, ge::proto::DT_FLOAT16},
{ge::DT_INT8, ge::proto::DT_INT8},
{ge::DT_UINT8, ge::proto::DT_UINT8},
{ge::DT_INT16, ge::proto::DT_INT16},
{ge::DT_UINT16, ge::proto::DT_UINT16},
{ge::DT_INT32, ge::proto::DT_INT32},
{ge::DT_INT64, ge::proto::DT_INT64},
{ge::DT_UINT32, ge::proto::DT_UINT32},
{ge::DT_UINT64, ge::proto::DT_UINT64},
{ge::DT_BOOL, ge::proto::DT_BOOL},
{ge::DT_DOUBLE, ge::proto::DT_DOUBLE},
{ge::DT_DUAL, ge::proto::DT_DUAL},
{ge::DT_DUAL_SUB_INT8, ge::proto::DT_DUAL_SUB_INT8},
{ge::DT_DUAL_SUB_UINT8, ge::proto::DT_DUAL_SUB_UINT8},
{ge::DT_COMPLEX64, ge::proto::DT_COMPLEX64},
{ge::DT_COMPLEX128, ge::proto::DT_COMPLEX128},
{ge::DT_QINT8, ge::proto::DT_QINT8},
{ge::DT_QINT16, ge::proto::DT_QINT16},
{ge::DT_QINT32, ge::proto::DT_QINT32},
{ge::DT_QUINT8, ge::proto::DT_QUINT8},
{ge::DT_QUINT16, ge::proto::DT_QUINT16},
{ge::DT_RESOURCE, ge::proto::DT_RESOURCE},
{ge::DT_STRING_REF, ge::proto::DT_STRING_REF},
{ge::DT_STRING, ge::proto::DT_STRING},
// key:ge datatype,value:proto datatype
{ge::DT_UNDEFINED, ge::proto::DT_UNDEFINED},
{ge::DT_FLOAT, ge::proto::DT_FLOAT},
{ge::DT_FLOAT16, ge::proto::DT_FLOAT16},
{ge::DT_INT8, ge::proto::DT_INT8},
{ge::DT_UINT8, ge::proto::DT_UINT8},
{ge::DT_INT16, ge::proto::DT_INT16},
{ge::DT_UINT16, ge::proto::DT_UINT16},
{ge::DT_INT32, ge::proto::DT_INT32},
{ge::DT_INT64, ge::proto::DT_INT64},
{ge::DT_UINT32, ge::proto::DT_UINT32},
{ge::DT_UINT64, ge::proto::DT_UINT64},
{ge::DT_BOOL, ge::proto::DT_BOOL},
{ge::DT_DOUBLE, ge::proto::DT_DOUBLE},
{ge::DT_DUAL, ge::proto::DT_DUAL},
{ge::DT_DUAL_SUB_INT8, ge::proto::DT_DUAL_SUB_INT8},
{ge::DT_DUAL_SUB_UINT8, ge::proto::DT_DUAL_SUB_UINT8},
{ge::DT_COMPLEX64, ge::proto::DT_COMPLEX64},
{ge::DT_COMPLEX128, ge::proto::DT_COMPLEX128},
{ge::DT_QINT8, ge::proto::DT_QINT8},
{ge::DT_QINT16, ge::proto::DT_QINT16},
{ge::DT_QINT32, ge::proto::DT_QINT32},
{ge::DT_QUINT8, ge::proto::DT_QUINT8},
{ge::DT_QUINT16, ge::proto::DT_QUINT16},
{ge::DT_RESOURCE, ge::proto::DT_RESOURCE},
{ge::DT_STRING_REF, ge::proto::DT_STRING_REF},
{ge::DT_STRING, ge::proto::DT_STRING},
};
} // namespace



+ 5
- 5
ge/common/ge/datatype_util.h View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -37,10 +37,10 @@ static const int32_t kGeSizeUint16 = sizeof(uint16_t);
static const int32_t kGeSizeUint32 = sizeof(uint32_t);

static std::map<ge::DataType, int32_t> CONST_OPDATA_TYPE_SIZE_MAP = {
{ge::DT_FLOAT, kGeSizeFloat}, {ge::DT_FLOAT16, kGeSizeHalfFloat}, {ge::DT_INT8, kGeSizeInt8},
{ge::DT_INT16, kGeSizeInt16}, {ge::DT_INT32, kGeSizeInt32}, {ge::DT_INT64, kGeSizeInt64},
{ge::DT_UINT8, kGeSizeUint8}, {ge::DT_UINT16, kGeSizeUint16}, {ge::DT_UINT32, kGeSizeUint32},
{ge::DT_UINT64, kGeSizeUint64}, {ge::DT_DOUBLE, kGeSizeDouble}, {ge::DT_BOOL, kGeSizeBool}};
{ge::DT_FLOAT, kGeSizeFloat}, {ge::DT_FLOAT16, kGeSizeHalfFloat}, {ge::DT_INT8, kGeSizeInt8},
{ge::DT_INT16, kGeSizeInt16}, {ge::DT_INT32, kGeSizeInt32}, {ge::DT_INT64, kGeSizeInt64},
{ge::DT_UINT8, kGeSizeUint8}, {ge::DT_UINT16, kGeSizeUint16}, {ge::DT_UINT32, kGeSizeUint32},
{ge::DT_UINT64, kGeSizeUint64}, {ge::DT_DOUBLE, kGeSizeDouble}, {ge::DT_BOOL, kGeSizeBool}};

class GE_FUNC_HOST_VISIBILITY GE_FUNC_DEV_VISIBILITY DataTypeUtil {
public:


+ 1
- 1
ge/common/ge/ge_util.h View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.


+ 3
- 3
ge/common/ge/op_tiling_manager.cc View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -45,8 +45,8 @@ std::string OpTilingManager::GetPath() {
if (opp_path_env != nullptr) {
char resolved_path[PATH_MAX];
if (realpath(opp_path_env, resolved_path) == NULL) {
ErrorManager::GetInstance().ATCReportErrMessage("E19024", {"env", "value", "situation"},
{"ASCEND_OPP_PATH", opp_path_env, "loading the tiling lib"});
ErrorManager::GetInstance().ATCReportErrMessage(
"E19024", {"env", "value", "situation"}, {"ASCEND_OPP_PATH", opp_path_env, "loading the tiling lib"});
GELOGE(PARAM_INVALID, "Failed load tiling lib as env 'ASCEND_OPP_PATH'[%s] is invalid path.", opp_path_env);
return std::string();
}


+ 2
- 1
ge/common/ge/op_tiling_manager.h View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -36,3 +36,4 @@ class OpTilingManager {
} // namespace ge

#endif // GE_COMMON_GE_OP_TILING_MANAGER_H_


+ 10
- 14
ge/common/ge/plugin_manager.cc View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -93,15 +93,13 @@ Status PluginManager::LoadSo(const string &path, const vector<string> &func_chec
std::vector<std::string> path_vec;
SplitPath(path, path_vec);
for (const auto &single_path : path_vec) {
GE_IF_BOOL_EXEC(single_path.length() >= PATH_MAX,
GELOGE(GE_PLGMGR_PATH_INVALID, "The shared library file path is too long!");
GE_IF_BOOL_EXEC(single_path.length() >= PATH_MAX, GELOGE(GE_PLGMGR_PATH_INVALID,
"The shared library file path is too long!");
continue);
// load break when number of loaded so reach maximum
if (num_of_loaded_so >= kMaxNumOfSo) {
GELOGW(
"The number of dynamic libraries loaded exceeds the kMaxNumOfSo,"
" and only the first %d shared libraries will be loaded.",
kMaxNumOfSo);
GELOGW("The number of dynamic libraries loaded exceeds the kMaxNumOfSo,"
" and only the first %d shared libraries will be loaded.", kMaxNumOfSo);
break;
}

@@ -182,9 +180,9 @@ Status PluginManager::ValidateSo(const string &file_path, int64_t size_of_loaded
// load continue if the total size of so reaches maximum when it is loaded
if (size_of_loaded_so + file_size > kMaxSizeOfLoadedSo) {
GELOGW(
"%s is skipped because the size of loaded share library reaches maximum if it is loaded! "
"(size: %ldB, size of loaded share library: %ldB, maximum: %dB)",
file_path.c_str(), file_size, size_of_loaded_so, kMaxSizeOfLoadedSo);
"%s is skipped because the size of loaded share library reaches maximum if it is loaded! "
"(size: %ldB, size of loaded share library: %ldB, maximum: %dB)",
file_path.c_str(), file_size, size_of_loaded_so, kMaxSizeOfLoadedSo);
return FAILED;
}

@@ -231,10 +229,8 @@ Status PluginManager::Load(const string &path, const vector<string> &func_check_

// load break when number of loaded so reach maximum
if (num_of_loaded_so >= kMaxNumOfSo) {
GELOGW(
"The number of dynamic libraries loaded exceeds the kMaxNumOfSo,"
" and only the first %d shared libraries will be loaded.",
kMaxNumOfSo);
GELOGW("The number of dynamic libraries loaded exceeds the kMaxNumOfSo,"
" and only the first %d shared libraries will be loaded.", kMaxNumOfSo);
break;
}



+ 1
- 1
ge/common/ge/plugin_manager.h View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.


+ 5
- 41
ge/common/ge/tbe_plugin_manager.cc View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -94,13 +94,6 @@ void TBEPluginManager::ProcessSoFullName(vector<string> &file_list, string &caff
full_name.compare(full_name.size() - caffe_parser_so_suff.size(), caffe_parser_so_suff.size(),
caffe_parser_so_suff) == 0) {
caffe_parser_path = full_name;
} else if ((full_name.size() >= aicpu_so_suff.size() &&
full_name.compare(full_name.size() - aicpu_so_suff.size(), aicpu_so_suff.size(), aicpu_so_suff) == 0) ||
(full_name.size() >= aicpu_host_so_suff.size() &&
full_name.compare(full_name.size() - aicpu_host_so_suff.size(), aicpu_host_so_suff.size(),
aicpu_host_so_suff) == 0)) {
// aicpu so, Put the file path into the omgcontext and save into the model in the builder stage.
domi::GetContext().aicpu_op_run_paths.push_back(full_name);
} else {
// Save parser so path into file_list vector
file_list.push_back(full_name);
@@ -193,8 +186,8 @@ void TBEPluginManager::LoadCustomOpLib() {
}
}

FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void TBEPluginManager::LoadPluginSo(
const std::map<string, string> &options) {
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY
void TBEPluginManager::LoadPluginSo(const std::map<string, string> &options) {
vector<string> file_list;
string caffe_parser_path;
std::string plugin_path;
@@ -230,39 +223,10 @@ FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void TBEPluginManager::LoadPlug
}
}

Status TBEPluginManager::CheckCustomAiCpuOpLib() {
std::vector<std::string> vec_op_type;

domi::OpRegistry::Instance()->GetOpTypeByImplyType(vec_op_type, domi::ImplyType::CUSTOM);
for (size_t i = 0; i < vec_op_type.size(); i++) {
bool aicpu_so_exist = false;
std::string ai_cpu_so_name = "lib" + vec_op_type[i] + "_aicpu.so";
for (size_t j = 0; j < domi::GetContext().aicpu_op_run_paths.size(); j++) {
string bin_file_path = domi::GetContext().aicpu_op_run_paths[j];
if (bin_file_path.size() >= ai_cpu_so_name.size() &&
bin_file_path.compare(bin_file_path.size() - ai_cpu_so_name.size(), ai_cpu_so_name.size(), ai_cpu_so_name) ==
0) {
aicpu_so_exist = true;
break;
}
}
if (!aicpu_so_exist) {
GELOGE(FAILED, "Can't find aicpu run so(%s), please check the plugin path!", ai_cpu_so_name.c_str());
return FAILED;
}
}
return SUCCESS;
}

FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY void TBEPluginManager::InitPreparation(
const std::map<string, string> &options) {
FMK_FUNC_HOST_VISIBILITY FMK_FUNC_DEV_VISIBILITY
void TBEPluginManager::InitPreparation(const std::map<string, string> &options) {
options_.insert(options.begin(), options.end());
// Load TBE plugin
TBEPluginManager::Instance().LoadCustomOpLib();
Status ret = CheckCustomAiCpuOpLib();
if (ret != SUCCESS) {
GELOGE(ret, "Check custom aicpu run so failed!");
return;
}
}
} // namespace ge

+ 6
- 7
ge/common/ge/tbe_plugin_manager.h View File

@@ -1,5 +1,5 @@
/**
* Copyright 2019-2020 Huawei Technologies Co., Ltd
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -32,23 +32,23 @@

namespace ge {
using SoHandlesVec = std::vector<void *>;
using std::function;
using std::map;
using std::string;
using std::vector;
using std::string;
using std::map;
using std::function;

class TBEPluginManager {
public:
Status Finalize();

// Get TBEPluginManager singleton instance
static TBEPluginManager &Instance();
static TBEPluginManager& Instance();

static string GetPath();

static void InitPreparation(const std::map<string, string> &options);

void LoadPluginSo(const std::map<string, string> &options);
void LoadPluginSo(const std::map< string, string> &options);

private:
TBEPluginManager() = default;
@@ -62,7 +62,6 @@ class TBEPluginManager {
static void GetPluginSoFileList(const string &path, vector<string> &file_list, string &caffe_parser_path);
static void GetCustomOpPath(std::string &customop_path);
void LoadCustomOpLib();
static Status CheckCustomAiCpuOpLib();

SoHandlesVec handles_vec_;
static std::map<string, string> options_;


+ 0
- 0
ge/common/ge_common.mk View File


+ 0
- 0
ge/common/ge_format_util.cc View File


+ 3
- 4
ge/common/helper/model_cache_helper.cc View File

@@ -1007,10 +1007,9 @@ Status ModelCacheHelper::RecoverVarAddrAndTensorDesc(const Json &json) const {
return PARAM_INVALID;
}
// Offset is needed by SaveVarVddr instead of logic address
ret =
VarManager::Instance(session_id_)
->SaveVarAddr(iter.first, tensor_addr_mgr.tensor_desc,
reinterpret_cast<uint8_t *>(reinterpret_cast<uintptr_t>(offset)), tensor_addr_mgr.memory_type);
ret = VarManager::Instance(session_id_)->SaveVarAddr(iter.first, tensor_addr_mgr.tensor_desc,
reinterpret_cast<uint8_t *>(reinterpret_cast<uintptr_t>(offset)),
tensor_addr_mgr.memory_type);
if (ret != SUCCESS) {
GELOGW("Fail to recover VarAddr or TensorDesc of var[%s].", iter.first.c_str());
return ret;


Some files were not shown because too many files changed in this diff

Loading…
Cancel
Save