protobuf test

CMakeLists.txt

@@ -42,6 +42,7 @@ include(${GE_SOURCE_DIR}/cmake/external_libs/json.cmake)
 include(${GE_SOURCE_DIR}/cmake/external_libs/eigen.cmake)
 include(${GE_SOURCE_DIR}/cmake/external_libs/gtest.cmake)
 include(${GE_SOURCE_DIR}/cmake/external_libs/protobuf.cmake)
+include(${GE_SOURCE_DIR}/cmake/external_libs/protoc.cmake)
 include(${GE_SOURCE_DIR}/cmake/external_libs/onnx.cmake)
 include(${GE_SOURCE_DIR}/cmake/external_libs/securec.cmake)
 set(CMAKE_SKIP_RPATH TRUE)

cmake/external_libs/protobuf.cmake

@@ -1,10 +1,15 @@
-if (NOT TARGET protobuf::protobuf)
-set(protobuf_USE_STATIC_LIBS ON)
-set(protobuf_CXXFLAGS "-Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -fstack-protector-all -D_FORTIFY_SOURCE=2 -O2")
-set(protobuf_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack")
-set(_ge_tmp_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
-string(REPLACE " -Wall" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
-string(REPLACE " -Werror" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
+if (HAVE_PROTOBUF)
+    return()
+endif()
+
+include(ExternalProject)
+include(GNUInstallDirs)
+
+if ((${CMAKE_INSTALL_PREFIX} STREQUAL /usr/local) OR
+    (${CMAKE_INSTALL_PREFIX} STREQUAL "C:/Program Files (x86)/ascend"))
+    set(CMAKE_INSTALL_PREFIX ${GE_CODE_DIR}/output CACHE STRING "path for install()" FORCE)
+    message(STATUS "No install prefix selected, default to ${CMAKE_INSTALL_PREFIX}.")
+endif()
 
 if (ENABLE_GITEE)
     set(REQ_URL "https://gitee.com/mirrors/protobuf_source/repository/archive/v3.8.0.tar.gz")
@@ -14,50 +19,47 @@ else()
     set(MD5 "3d9e32700639618a4d2d342c99d4507a")
 endif ()
 
-graphengine_add_pkg(protobuf
-        VER 3.8.0
-        LIBS protobuf
-        EXE protoc
-        URL ${REQ_URL}
-        MD5 ${MD5}
-        CMAKE_PATH ../cmake/
-        CMAKE_OPTION -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF)
-set(CMAKE_CXX_FLAGS ${_ge_tmp_CMAKE_CXX_FLAGS})
-endif()
-add_library(graphengine::protobuf ALIAS protobuf::protobuf)
-set(PROTOBUF_LIBRARY protobuf::protobuf)
-include_directories(${protobuf_INC})
-include_directories(${protobuf_DIRPATH}/src)
-
-function(ge_protobuf_generate comp c_var h_var)
-    if(NOT ARGN)
-        message(SEND_ERROR "Error: ge_protobuf_generate() called without any proto files")
-        return()
-    endif()
-
-    set(${c_var})
-    set(${h_var})
-
-    foreach(file ${ARGN})
-        get_filename_component(abs_file ${file} ABSOLUTE)
-        get_filename_component(file_name ${file} NAME_WE)
-        get_filename_component(file_dir ${abs_file} PATH)
-
-        list(APPEND ${c_var} "${CMAKE_BINARY_DIR}/proto/${comp}/proto/${file_name}.pb.cc")
-        list(APPEND ${h_var} "${CMAKE_BINARY_DIR}/proto/${comp}/proto/${file_name}.pb.h")
-
-        add_custom_command(
-                OUTPUT "${CMAKE_BINARY_DIR}/proto/${comp}/proto/${file_name}.pb.cc"
-                "${CMAKE_BINARY_DIR}/proto/${comp}/proto/${file_name}.pb.h"
-                WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
-                COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}/proto/${comp}/proto"
-                COMMAND protobuf::protoc -I${file_dir} --cpp_out=${CMAKE_BINARY_DIR}/proto/${comp}/proto ${abs_file}
-                DEPENDS protobuf::protoc ${abs_file}
-                COMMENT "Running C++ protocol buffer compiler on ${file}" VERBATIM )
-    endforeach()
-
-    set_source_files_properties(${${c_var}} ${${h_var}} PROPERTIES GENERATED TRUE)
-    set(${c_var} ${${c_var}} PARENT_SCOPE)
-    set(${h_var} ${${h_var}} PARENT_SCOPE)
-
-endfunction()
+set(protobuf_CXXFLAGS "-Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -fstack-protector-all -D_FORTIFY_SOURCE=2 -D_GLIBCXX_USE_CXX11_ABI=0 -O2 -Dgoogle=ascend_private")
+set(protobuf_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack")
+ExternalProject_Add(protobuf_build
+                    URL https://github.com/protocolbuffers/protobuf/archive/v3.8.0.tar.gz
+                    CONFIGURE_COMMAND ${CMAKE_COMMAND}
+                    -Dprotobuf_WITH_ZLIB=OFF
+                    -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR}
+                    -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+                    -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
+                    -DCMAKE_LINKER=${CMAKE_LINKER}
+                    -DCMAKE_AR=${CMAKE_AR}
+                    -DCMAKE_RANLIB=${CMAKE_RANLIB}
+                    -DLIB_PREFIX=ascend_
+                    -Dprotobuf_BUILD_TESTS=OFF -DBUILD_SHARED_LIBS=ON -DCMAKE_CXX_FLAGS=${protobuf_CXXFLAGS} -DCMAKE_CXX_LDFLAGS=${protobuf_LDFLAGS} -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}/protobuf <SOURCE_DIR>/cmake
+                    BUILD_COMMAND $(MAKE)
+                    INSTALL_COMMAND $(MAKE) install
+                    EXCLUDE_FROM_ALL TRUE
+)
+include(GNUInstallDirs)
+
+set(PROTOBUF_SHARED_PKG_DIR ${CMAKE_INSTALL_PREFIX}/protobuf)
+
+add_library(ascend_protobuf SHARED IMPORTED)
+
+file(MAKE_DIRECTORY ${PROTOBUF_SHARED_PKG_DIR}/include)
+
+set_target_properties(ascend_protobuf PROPERTIES
+                      IMPORTED_LOCATION ${PROTOBUF_SHARED_PKG_DIR}/${CMAKE_INSTALL_LIBDIR}/libascend_protobuf.so
+)
+
+target_include_directories(ascend_protobuf INTERFACE ${PROTOBUF_SHARED_PKG_DIR}/include)
+
+set(INSTALL_BASE_DIR "")
+set(INSTALL_LIBRARY_DIR lib)
+
+install(FILES ${PROTOBUF_SHARED_PKG_DIR}/${CMAKE_INSTALL_LIBDIR}/ascend_protobuf.so.3.8.0.0 OPTIONAL
+        DESTINATION ${INSTALL_LIBRARY_DIR})
+install(FILES ${PROTOBUF_SHARED_PKG_DIR}/${CMAKE_INSTALL_LIBDIR}/ascend_protobuf.so OPTIONAL
+        DESTINATION ${INSTALL_LIBRARY_DIR})
+
+add_dependencies(ascend_protobuf protobuf_build)
+
+#set(HAVE_PROTOBUF TRUE CACHE BOOL "protobuf build add")
+set(HAVE_PROTOBUF TRUE)

cmake/external_libs/protoc.cmake

@@ -0,0 +1,112 @@
+if (HAVE_PROTOC)
+    return()
+endif()
+
+include(ExternalProject)
+include(GNUInstallDirs)
+#set(CMAKE_INSTALL_PREFIX ${GE_CODE_DIR}/output)
+
+if ((${CMAKE_INSTALL_PREFIX} STREQUAL /usr/local) OR
+    (${CMAKE_INSTALL_PREFIX} STREQUAL "C:/Program Files (x86)/ascend"))
+    set(CMAKE_INSTALL_PREFIX ${GE_CODE_DIR}/output CACHE STRING "path for install()" FORCE)
+    message(STATUS "No install prefix selected, default to ${CMAKE_INSTALL_PREFIX}.")
+endif()
+
+if (ENABLE_GITEE)
+    set(REQ_URL "https://gitee.com/mirrors/protobuf_source/repository/archive/v3.8.0.tar.gz")
+    set(MD5 "eba86ae9f07ba5cfbaf8af3bc4e84236")
+else()
+    set(REQ_URL "https://github.com/protocolbuffers/protobuf/archive/v3.8.0.tar.gz")
+    set(MD5 "3d9e32700639618a4d2d342c99d4507a")
+endif ()
+
+
+set(protobuf_CXXFLAGS "-Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -fstack-protector-all -D_FORTIFY_SOURCE=2 -D_GLIBCXX_USE_CXX11_ABI=0 -O2")
+set(protobuf_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack")
+ExternalProject_Add(protoc_build
+                    URL ${REQ_URL}
+                    #URL /home/txd/workspace/linux_cmake/pkg/protobuf-3.8.0.tar.gz
+                    #SOURCE_DIR ${GE_CODE_DIR}/../third_party/protobuf/src/protobuf-3.8.0
+                    CONFIGURE_COMMAND ${CMAKE_COMMAND} -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF -DCMAKE_CXX_FLAGS=${protobuf_CXXFLAGS} -DCMAKE_CXX_LDFLAGS=${protobuf_LDFLAGS} -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}/protoc <SOURCE_DIR>/cmake
+                    BUILD_COMMAND $(MAKE)
+                    INSTALL_COMMAND $(MAKE) install
+                    EXCLUDE_FROM_ALL TRUE
+)
+
+set(PROTOC_PKG_DIR ${CMAKE_INSTALL_PREFIX}/protoc)
+
+set(protoc_EXECUTABLE ${PROTOC_PKG_DIR}/${CMAKE_INSTALL_BINDIR}/protoc)
+
+function(protobuf_generate comp c_var h_var)
+    if(NOT ARGN)
+        message(SEND_ERROR "Error: protobuf_generate() called without any proto files")
+        return()
+    endif()
+    set(${c_var})
+    set(${h_var})
+
+    foreach(file ${ARGN})
+        get_filename_component(abs_file ${file} ABSOLUTE)
+        get_filename_component(file_name ${file} NAME_WE)
+        get_filename_component(file_dir ${abs_file} PATH)
+        get_filename_component(parent_subdir ${file_dir} NAME)
+
+        if("${parent_subdir}" STREQUAL "proto")
+            set(proto_output_path ${CMAKE_BINARY_DIR}/proto/${comp}/proto)
+        else()
+            set(proto_output_path ${CMAKE_BINARY_DIR}/proto/${comp}/proto/${parent_subdir})
+        endif()
+        list(APPEND ${c_var} "${proto_output_path}/${file_name}.pb.cc")
+        list(APPEND ${h_var} "${proto_output_path}/${file_name}.pb.h")
+
+        add_custom_command(
+                OUTPUT "${proto_output_path}/${file_name}.pb.cc" "${proto_output_path}/${file_name}.pb.h"
+                WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+                COMMAND ${CMAKE_COMMAND} -E make_directory "${proto_output_path}"
+                COMMAND ${protoc_EXECUTABLE} -I${file_dir} --cpp_out=${proto_output_path} ${abs_file}
+                DEPENDS protoc_build ${abs_file}
+                COMMENT "Running C++ protocol buffer compiler on ${file}" VERBATIM )
+    endforeach()
+
+    set_source_files_properties(${${c_var}} ${${h_var}} PROPERTIES GENERATED TRUE)
+    set(${c_var} ${${c_var}} PARENT_SCOPE)
+    set(${h_var} ${${h_var}} PARENT_SCOPE)
+
+endfunction()
+
+function(protobuf_generate_py comp py_var)
+    if(NOT ARGN)
+        message(SEND_ERROR "Error: protobuf_generate_py() called without any proto files")
+        return()
+    endif()
+    set(${py_var})
+
+    foreach(file ${ARGN})
+        get_filename_component(abs_file ${file} ABSOLUTE)
+        get_filename_component(file_name ${file} NAME_WE)
+        get_filename_component(file_dir ${abs_file} PATH)
+        get_filename_component(parent_subdir ${file_dir} NAME)
+
+        if("${parent_subdir}" STREQUAL "proto")
+            set(proto_output_path ${CMAKE_BINARY_DIR}/proto/${comp}/proto)
+        else()
+            set(proto_output_path ${CMAKE_BINARY_DIR}/proto/${comp}/proto/${parent_subdir})
+        endif()
+        list(APPEND ${py_var} "${proto_output_path}/${file_name}_pb2.py")
+
+        add_custom_command(
+                OUTPUT "${proto_output_path}/${file_name}_pb2.py"
+                WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+                COMMAND ${CMAKE_COMMAND} -E make_directory "${proto_output_path}"
+                COMMAND ${protoc_EXECUTABLE} -I${file_dir} --python_out=${proto_output_path} ${abs_file}
+                DEPENDS protoc_build ${abs_file}
+                COMMENT "Running PYTHON protocol buffer compiler on ${file}" VERBATIM )
+    endforeach()
+
+    set_source_files_properties(${${py_var}} PROPERTIES GENERATED TRUE)
+    set(${py_var} ${${py_var}} PARENT_SCOPE)
+
+endfunction()
+
+#set(HAVE_PROTOC TRUE CACHE BOOL "protoc build add")
+set(HAVE_PROTOC TRUE)

src/common/graph/CMakeLists.txt

@@ -31,8 +31,8 @@ file(GLOB_RECURSE ONNX_PROTO_LIST RELATIVE ${CMAKE_CURRENT_LIST_DIR}
         "${onnx_INC}/onnx/onnx.proto"
         )
 
-ge_protobuf_generate(ge PROTO_SRCS PROTO_HDRS ${PROTO_LIST})
-ge_protobuf_generate(ge PROTO_ONNX_SRCS PROTO_ONNX_HDRS ${ONNX_PROTO_LIST})
+protobuf_generate(ge PROTO_SRCS PROTO_HDRS ${PROTO_LIST})
+protobuf_generate(ge PROTO_ONNX_SRCS PROTO_ONNX_HDRS ${ONNX_PROTO_LIST})
 
 # need to remove dependencies on pb files later
 file(GLOB SRC_LIST RELATIVE ${CMAKE_CURRENT_LIST_DIR}

src/ge/CMakeLists.txt

@@ -33,9 +33,9 @@ file(GLOB PROTO_HEADER_LIST RELATIVE ${CMAKE_CURRENT_LIST_DIR}
         "../proto/op_mapping_info.proto"
         "../proto/dump_task.proto"
         )
-ge_protobuf_generate(ge PROTO_SRCS PROTO_HDRS ${PROTO_LIST})
-ge_protobuf_generate(ge PROTO_CLIENT_SRCS PROTO_CLIENT_HDRS ${PROTO_CLIENT_LIST})
-ge_protobuf_generate(ge PROTO_HEADER_SRCS PROTO_HEADER_HDRS ${PROTO_HEADER_LIST})
+protobuf_generate(ge PROTO_SRCS PROTO_HDRS ${PROTO_LIST})
+protobuf_generate(ge PROTO_CLIENT_SRCS PROTO_CLIENT_HDRS ${PROTO_CLIENT_LIST})
+protobuf_generate(ge PROTO_HEADER_SRCS PROTO_HEADER_HDRS ${PROTO_HEADER_LIST})
 # include directories
 include_directories(${CMAKE_CURRENT_LIST_DIR})
 include_directories(${GE_SOURCE_DIR})

src/ge/client/CMakeLists.txt

@@ -32,8 +32,8 @@ file(GLOB SRC_LIST RELATIVE ${CMAKE_CURRENT_LIST_DIR}
         "ge_prof.cc"
         )
 
-ge_protobuf_generate(ge PROTO_SRCS PROTO_HDRS ${PROTO_LIST})
-ge_protobuf_generate(ge PROTO_HEADER_SRCS PROTO_HEADER_HDRS ${PROTO_HEADER_LIST})
+protobuf_generate(ge PROTO_SRCS PROTO_HDRS ${PROTO_LIST})
+protobuf_generate(ge PROTO_HEADER_SRCS PROTO_HEADER_HDRS ${PROTO_HEADER_LIST})
 
 # include directories
 include_directories(${CMAKE_CURRENT_LIST_DIR})

src/ge/common/CMakeLists.txt

@@ -67,7 +67,7 @@ file(GLOB SRC_LIST RELATIVE ${CMAKE_CURRENT_LIST_DIR}
         "util.cc"
         )
 
-ge_protobuf_generate(ge PROTO_SRCS PROTO_HDRS ${PROTO_LIST})
+protobuf_generate(ge PROTO_SRCS PROTO_HDRS ${PROTO_LIST})
 
 # include directories
 include_directories(${CMAKE_CURRENT_LIST_DIR})

src/ge/executor/CMakeLists.txt

@@ -87,7 +87,7 @@ file(GLOB SRC_LIST RELATIVE ${CMAKE_CURRENT_LIST_DIR}
         "../single_op/task/tbe_task_builder.cc"
         )
 
-ge_protobuf_generate(ge PROTO_SRCS PROTO_HDRS ${PROTO_LIST})
+protobuf_generate(ge PROTO_SRCS PROTO_HDRS ${PROTO_LIST})
 
 # include directories
 include_directories(${CMAKE_CURRENT_LIST_DIR})

src/ge/ge_local_engine/CMakeLists.txt

@@ -25,7 +25,7 @@ file(GLOB SRC_LIST RELATIVE ${CMAKE_CURRENT_LIST_DIR}
         "ops_kernel_store/op/*.cc"
         )
 
-ge_protobuf_generate(ge PROTO_SRCS PROTO_HDRS ${PROTO_LIST})
+protobuf_generate(ge PROTO_SRCS PROTO_HDRS ${PROTO_LIST})
 
 # include directories
 include_directories(${CMAKE_CURRENT_LIST_DIR})