hummingbird
/
graphengine
Not watched
2
0
Fork 0
Code
Releases 13 Wiki Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain
Browse Source

add tool-chains for graphengine 

create container name by different code path

remove no use console for update

add ge config command for server addr config

add readme for ge config

add readme for ge config
tags/v1.3.0
wei-gangqiang WeiGangqiang 3 years ago
parent
f18934c067
commit
f5dc59ad7d
18 changed files with 6661 additions and 61 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +9
    -0
      .gitignore
  2. +0
    -0
      build.sh
  3. +25
    -0
      scripts/build/ge_build.sh
  4. +38
    -38
      scripts/check_clang_format.sh
  5. +90
    -0
      scripts/clean/ge_clean.sh
  6. +115
    -0
      scripts/config/ge_config.sh
  7. +136
    -0
      scripts/cov/ge_cov.sh
  8. +2639
    -0
      scripts/docs/Doxyfile_all
  9. +2638
    -0
      scripts/docs/Doxyfile_brief
  10. +87
    -0
      scripts/docs/ge_docs.sh
  11. +42
    -0
      scripts/env/Dockerfile
  12. +146
    -0
      scripts/env/ge_env.sh
  13. +25
    -23
      scripts/format/ge_format.sh
  14. +77
    -0
      scripts/ge.sh
  15. +331
    -0
      scripts/readme.md
  16. +80
    -0
      scripts/test/ge_test.sh
  17. +47
    -0
      scripts/update/deps_config.sh
  18. +136
    -0
      scripts/update/ge_update.sh

+ 9
- 0
.gitignore View File

@@ -3,8 +3,17 @@
/output
/prebuilts
/cov
/deps
.autotools
.project
.cproject
.settings/
/tests/frm/
*.ir
*.out
*.DS_Store
.DS_Store
server_config.sh

# Dynamic libraries
# *.so


+ 0
- 0
build.sh View File


+ 25
- 0
scripts/build/ge_build.sh View File

@@ -0,0 +1,25 @@
#!/bin/bash
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================

set -e
export PROJECT_HOME=${PROJECT_HOME:-$(dirname "$0")/../}
function main(){
${PROJECT_HOME}/build.sh "$@"
}

main "$@"

set +e

+ 38
- 38
scripts/check_clang_format.sh View File

@@ -1,5 +1,5 @@
#!/bin/bash
# Copyright 2019-2020 Huawei Technologies Co., Ltd
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================

set -e

CLANG_FORMAT=$(which clang-format) || (echo "Please install 'clang-format' tool first"; exit 1)
@@ -25,10 +24,10 @@ if [[ "${version}" -lt "8" ]]; then
fi

CURRENT_PATH=$(pwd)
SCRIPTS_PATH=$(dirname "$0")
PROJECT_HOME=${PROJECT_HOME:-$(dirname "$0")/../}

echo "CURRENT_PATH=$CURRENT_PATH"
echo "SCRIPTS_PATH=$SCRIPTS_PATH"
echo "PROJECT_HOME=$PROJECT_HOME"

# print usage message
function usage()
@@ -81,45 +80,46 @@ function checkopts()
checkopts "$@"

# switch to project root path, which contains clang-format config file '.clang-format'
cd "${SCRIPTS_PATH}/.." || exit 1
pushd "${CURRENT_PATH}"
CHECK_LIST_FILE='__checked_files_list__'
cd "${PROJECT_HOME}" || exit 1

CHECK_LIST_FILE='__checked_files_list__'
if [ "X${mode}" == "Xall" ]; then
find src -type f -name "*" | grep "\.h$\|\.cc$" > "${CHECK_LIST_FILE}" || true
find inc -type f -name "*" | grep "\.h$\|\.cc$" >> "${CHECK_LIST_FILE}" || true
elif [ "X${mode}" == "Xchanged" ]; then
# --diff-filter=ACMRTUXB will ignore deleted files in commit
git diff --diff-filter=ACMRTUXB --name-only | grep "^inc\|^src" | grep "\.h$\|\.cc$" > "${CHECK_LIST_FILE}" || true
else # "X${mode}" == "Xlastcommit"
git diff --diff-filter=ACMRTUXB --name-only HEAD~ HEAD | grep "^inc\|^src" | grep "\.h$\|\.cc$" > "${CHECK_LIST_FILE}" || true
fi

if [ "X${mode}" == "Xall" ]; then
find src -type f -name "*" | grep "\.h$\|\.cc$" > "${CHECK_LIST_FILE}" || true
find inc -type f -name "*" | grep "\.h$\|\.cc$" >> "${CHECK_LIST_FILE}" || true
elif [ "X${mode}" == "Xchanged" ]; then
# --diff-filter=ACMRTUXB will ignore deleted files in commit
git diff --diff-filter=ACMRTUXB --name-only | grep "^inc\|^src" | grep "\.h$\|\.cc$" > "${CHECK_LIST_FILE}" || true
else # "X${mode}" == "Xlastcommit"
git diff --diff-filter=ACMRTUXB --name-only HEAD~ HEAD | grep "^inc\|^src" | grep "\.h$\|\.cc$" > "${CHECK_LIST_FILE}" || true
fi
CHECK_RESULT_FILE=__code_format_check_result__
echo "0" > "$CHECK_RESULT_FILE"

CHECK_RESULT_FILE=__code_format_check_result__
echo "0" > "$CHECK_RESULT_FILE"
# check format of files modified in the lastest commit
while read line; do
BASE_NAME=$(basename "${line}")
TEMP_FILE="__TEMP__${BASE_NAME}"
cp "${line}" "${TEMP_FILE}"
${CLANG_FORMAT} -i "${TEMP_FILE}"
set +e
diff "${TEMP_FILE}" "${line}"
ret=$?
set -e
rm "${TEMP_FILE}"
if [[ "${ret}" -ne 0 ]]; then
echo "File ${line} is not formated, please format it."
echo "1" > "${CHECK_RESULT_FILE}"
break
fi
done < "${CHECK_LIST_FILE}"

# check format of files modified in the lastest commit
while read line; do
BASE_NAME=$(basename "${line}")
TEMP_FILE="__TEMP__${BASE_NAME}"
cp "${line}" "${TEMP_FILE}"
${CLANG_FORMAT} -i "${TEMP_FILE}"
set +e
diff "${TEMP_FILE}" "${line}"
ret=$?
set -e
rm "${TEMP_FILE}"
if [[ "${ret}" -ne 0 ]]; then
echo "File ${line} is not formated, please format it."
echo "1" > "${CHECK_RESULT_FILE}"
break
fi
done < "${CHECK_LIST_FILE}"
result=$(cat "${CHECK_RESULT_FILE}")
rm "${CHECK_RESULT_FILE}"
rm "${CHECK_LIST_FILE}"
popd

result=$(cat "${CHECK_RESULT_FILE}")
rm "${CHECK_RESULT_FILE}"
rm "${CHECK_LIST_FILE}"
cd "${CURRENT_PATH}" || exit 1
if [[ "X${result}" == "X0" ]]; then
echo "Check PASS: specified files are well formated!"
fi


+ 90
- 0
scripts/clean/ge_clean.sh View File

@@ -0,0 +1,90 @@
#!/bin/bash
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================

set -e

PROJECT_HOME=${PROJECT_HOME:-$(dirname "$0")/../../}

function help(){
cat <<-EOF
Usage: ge clean [OPTIONS]

Options:
-b, --build Clean build dir
-d, --docs Clean generate docs
-i, --install Clean dependenices
-a, --all Clean all
-h, --help
EOF

}

function clean_relative_dir(){
rm -rf "${PROJECT_HOME}/${1:-output}"
}

function parse_args(){
parsed_args=$(getopt -a -o bdiah --long build,docs,install,all,help -- "$@") || {
help
exit 1
}

if [ $# -lt 1 ]; then
clean_relative_dir "build"
clean_relative_dir "output"
exit 1
fi

eval set -- "$parsed_args"
while true; do
case "$1" in
-b | --build)
clean_relative_dir "build"
clean_relative_dir "output"
;;
-d | --docs)
clean_relative_dir "docs/doxygen"
;;
-i | --install)
clean_relative_dir "deps"
;;
-a | --all)
clean_relative_dir "deps"
clean_relative_dir "build"
clean_relative_dir "output"
clean_relative_dir "docs/doxygen"
;;
-h | --help)
help
;;
--)
shift; break;
;;
*)
help; exit 1
;;
esac
shift
done
}

function main(){
parse_args "$@"
}

main "$@"

set +e

+ 115
- 0
scripts/config/ge_config.sh View File

@@ -0,0 +1,115 @@
#!/bin/bash
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================

set -e

PROJECT_HOME=${PROJECT_HOME:-$(dirname "$0")/../../}
PROJECT_HOME=$(cd $PROJECT_HOME || return; pwd)

function help(){
cat <<-EOF
Usage: ge config [OPTIONS]

update server config for ge, you need input all config info (ip, user, password)

Options:
-i, --ip Config ip config
-u, --user Config user name
-p, --password Config password
-h, --help

Example: ge config -i=121.36.**.** -u=asc**, -p=Asc***\#@\!\$ (Need add escape character \ before special charactor $、#、!)

EOF

}

function write_config_file(){
local IP=$1
local USER=$2
local PASSWORD=$3
if [[ -z "$IP" ]] || [[ -z "$USER" ]] || [[ -z "$USER" ]]; then
echo "You need input all info （ip, user,password）obout server config !!!"
help
exit 1
fi

local PASSWORD=${PASSWORD//!/\\!}
local PASSWORD=${PASSWORD//#/\\#}
local PASSWORD=${PASSWORD/\$/\\\$}
local SERVER_CONFIG_FILE=${PROJECT_HOME}/scripts/config/server_config.sh
[ -n "${SERVER_CONFIG_FILE}" ] && rm -rf "${SERVER_CONFIG_FILE}"

cat>${SERVER_CONFIG_FILE}<<-EOF
SERVER_PATH=http://${IP}/package/etrans
DEP_USER=${USER}
DEP_PASSWORD=${PASSWORD}

EOF

}



function parse_args(){
parsed_args=$(getopt -a -o i::u::p::h --long ip::,user::,password::,help -- "$@") || {
help
exit 1
}

if [ $# -lt 1 ]; then
help
exit 1
fi
local IP=
local USER=
local PASSWORD=

eval set -- "$parsed_args"
while true; do
case "$1" in
-i | --ip)
IP=$2
;;
-u | --user)
USER=$2
;;
-p | --password)
PASSWORD=$2
;;
-h | --help)
help; exit;
;;
--)
shift; break;
;;
*)
help; exit 1
;;
esac
shift 2
done

write_config_file $IP $USER $PASSWORD
}

function main(){
parse_args "$@"
}

main "$@"

set +e

+ 136
- 0
scripts/cov/ge_cov.sh View File

@@ -0,0 +1,136 @@
#!/bin/bash
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================

set -e


function help(){
cat <<-EOF
Usage: ge cov [OPTIONS]

Options:
-a, --all Full coverage
-i, --increment Increment coverage
-d, --directory Coverage of directory
-h, --help
EOF

}

PROJECT_HOME=${PROJECT_HOME:-$(dirname "$0")/../../}
PROJECT_HOME=$(cd $PROJECT_HOME || return; pwd)

ALL_COV_GEN_PATH=${PROJECT_HOME}/cov/all
DIFF_FILE_PATH=${PROJECT_HOME}/cov/diff
DIFF_FILE_NAME=${DIFF_FILE_PATH}/inc_change_diff.txt

function process_diff_format(){
sed -i "s/--- a/--- \/code\/Turing\/graphEngine/g" ${DIFF_FILE_NAME}
sed -i "s/+++ b/+++ \/code\/Turing\/graphEngine/g" ${DIFF_FILE_NAME}
}


function add_cov_generate(){
addlcov --diff ${ALL_COV_GEN_PATH}/coverage.info ${DIFF_FILE_NAME} -o ${PROJECT_HOME}/cov/diff/inc_coverage.info
}

function gen_add_cov_html(){
genhtml --prefix ${PROJECT_HOME} -o ${PROJECT_HOME}/cov/diff/html ${PROJECT_HOME}/cov/diff/inc_coverage.info --legend -t CHG --no-branch-coverage --no-function-coverage
}

function increment_cov_for_directory(){
[ -n "${DIFF_FILE_PATH}" ] && rm -rf "${DIFF_FILE_PATH}"
mkdir -p ${DIFF_FILE_PATH}
git diff HEAD -- $1 >>${DIFF_FILE_NAME}
process_diff_format
add_cov_generate
gen_add_cov_html
}

function run_all_coverage(){
[ -n "${ALL_COV_GEN_PATH}" ] && rm -rf ${ALL_COV_GEN_PATH}
mkdir -p ${ALL_COV_GEN_PATH}
pushd "${PWD}" >/dev/null
cd ${PROJECT_HOME}
lcov -c -d build/tests/ut/ge -d build/tests/ut/common/graph/ -o ${ALL_COV_GEN_PATH}/tmp.info
lcov -r ${ALL_COV_GEN_PATH}/tmp.info '*/output/*' '*/build/opensrc/*' '*/build/proto/*' '*/third_party/*' '*/tests/*' '/usr/local/*' '/usr/include/*' '*/metadef/*' '*/parser/*' -o ${ALL_COV_GEN_PATH}/coverage.info
cd ${ALL_COV_GEN_PATH}
genhtml coverage.info
popd >/dev/null
}

function do_coverage_run(){
local cov_mode=$1
local directory_dir=$2
run_all_coverage

if [ "$cov_mode" = "all" ]; then
exit 1
elif [ -n "$directory_dir" ]; then
increment_cov_for_directory $directory_dir
else
increment_cov_for_directory "ge"
fi
}

function parse_args(){
parsed_args=$(getopt -a -o aid::h --long all,increment,directory::,help -- "$@") || {
help
exit 1
}

if [ $# -lt 1 ]; then
run_all_coverage
exit 1
fi

local cov_mode="increment"
local directory_dir=
eval set -- "$parsed_args"
while true; do
case "$1" in
-a | --all)
cov_mode="all"
;;
-i | --increment)
;;
-d | --directory)
directory_dir=$2
shift
;;
-h | --help)
help; exit 1;
;;
--)
shift; break;
;;
*)
help; exit 1;
;;
esac
shift
done
do_coverage_run $cov_mode $directory_dir
}

function main(){
parse_args "$@"
}

main "$@"

set +e

+ 2639
- 0
scripts/docs/Doxyfile_all
File diff suppressed because it is too large
View File


+ 2638
- 0
scripts/docs/Doxyfile_brief
File diff suppressed because it is too large
View File


+ 87
- 0
scripts/docs/ge_docs.sh View File

@@ -0,0 +1,87 @@
#!/bin/bash
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================

set -e

function help(){
cat <<-EOF
Usage: ge docs [OPTIONS]

Options:
-b, --brief Build brief docs
-a, --all Build all docs
-h, --help
EOF

}

PROJECT_HOME=${PROJECT_HOME:-$(dirname "$0")/../../}
PROJECT_HOME=$(cd $PROJECT_HOME || return; pwd)
BRIEF_DOXYFILE_PATH=${PROJECT_HOME}/scripts/docs/Doxyfile_brief
ALL_DOXYFILE_PATH=${PROJECT_HOME}/scripts/docs/Doxyfile_all


function build_brief_docs(){
rm -rf "${PROJECT_HOME}/docs/doxygen"
doxygen ${BRIEF_DOXYFILE_PATH}
}

function build_all_docs(){
rm -rf "${PROJECT_HOME}/docs/doxygen"
doxygen ${ALL_DOXYFILE_PATH}
}

function parse_args(){
parsed_args=$(getopt -a -o bah --long brief,all,help -- "$@") || {
help
exit 1
}

if [ $# -lt 1 ]; then
build_all_docs
exit 1
fi

eval set -- "$parsed_args"
while true; do
case "$1" in
-b | --brief)
build_brief_docs
;;
-a | --all)
build_all_docs
;;
-h | --help)
help; exit 1;
;;
--)
shift; break;
;;
*)
help; exit 1;
;;
esac
shift
done
}

function main(){
parse_args "$@"
}

main "$@"

set +e

+ 42
- 0
scripts/env/Dockerfile View File

@@ -0,0 +1,42 @@
# this dockerfile used for graphengine build
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================

FROM ubuntu:18.04
RUN apt-get update \
&& apt-get install -y git g++ wget unzip clang-format-9 build-essential lcov vim

# install for doxygen
RUN apt-get install -y graphviz doxygen

# install for graph ensy engine
RUN cpan install -y Graph::Easy

RUN wget https://cmake.org/files/v3.16/cmake-3.16.7-Linux-x86_64.tar.gz

RUN mkdir -p /opt/cmake-3.16.7 \
&& tar -xvf cmake-3.16.7-Linux-x86_64.tar.gz -C /opt/cmake-3.16.7 --strip-components=1 \
&& ln -sf /opt/cmake-3.16.7/bin/* /usr/bin/ \
&& mv /usr/bin/clang-format-9 /usr/bin/clang-format

RUN wget https://github.com/ccup/lcov/archive/refs/tags/add_lcov.tar.gz -O add_lcov.tar.gz \
&& mkdir -p /opt/addlcov1.0.0 \
&& tar -xvf add_lcov.tar.gz -C /opt/addlcov1.0.0 \
&& mv /opt/addlcov1.0.0/lcov-add_lcov/bin/lcov /usr/bin/addlcov

ENV PROJECT_HOME=/code/Turing/graphEngine

RUN echo "alias ge=/code/Turing/graphEngine/scripts/ge.sh">>~/.bashrc


+ 146
- 0
scripts/env/ge_env.sh View File

@@ -0,0 +1,146 @@
#!/bin/bash
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================

set -e

PROJECT_HOME=${PROJECT_HOME:-$(dirname "$0")/../../}
MOUNT_PROJECT_HOME=$(cd $PROJECT_HOME || return; pwd)

DOCKER_BUILD_ENV_NAME=${MOUNT_PROJECT_HOME#*/}
DOCKER_BUILD_ENV_NAME=${DOCKER_BUILD_ENV_NAME//\//\_}
DOCKER_IMAGE_TAG=ge_build_env.1.0.6
DOCKER_IAMGE_NAME=joycode2art/turing
DOCKER_FULL_IMAGE_NAME=${DOCKER_IAMGE_NAME}:${DOCKER_IMAGE_TAG}

if [ "$(uname)" == "Darwin" ]; then
#running on Mac OS
docker_cmd=docker
MOUNT_PROJECT_HOME=${MOUNT_PROJECT_HOME}
docker_work_dir=/code/Turing/graphEngine
docker_bash_dir=/bin/bash
elif [ "$(expr substr "$(uname -s)" 1 10)" == "MINGW32_NT" ] || [ "$(expr substr "$(uname -s)" 1 10)" == "MINGW64_NT" ]; then
#running on Windows
docker_cmd="winpty docker"
MOUNT_PROJECT_HOME=/${MOUNT_PROJECT_HOME}
docker_work_dir=//code/Turing/graphEngine
docker_bash_dir=//bin/bash
elif [ "$(expr substr "$(uname -s)" 1 5)" == "Linux" ]; then
#running on Linux
docker_cmd=docker
MOUNT_PROJECT_HOME=${PROJECT_HOME}
docker_work_dir=/code/Turing/graphEngine
docker_bash_dir=/bin/bash
fi

function build_docker_image(){
if test -z "$(docker images |grep ${DOCKER_IAMGE_NAME} | grep ${DOCKER_IMAGE_TAG})"; then
$docker_cmd build -t ${DOCKER_FULL_IMAGE_NAME} ${PROJECT_HOME}/scripts/env
else
echo "docker image for graph engine build is build ok...."
fi
}

function pull_docker_image(){
$docker_cmd pull $DOCKER_FULL_IMAGE_NAME
}

function enter_docker_env(){
if test -z "$(docker images |grep ${DOCKER_IAMGE_NAME} | grep ${DOCKER_IMAGE_TAG})"; then
echo "please run 'ge env --pull' to download images first!"
elif test -z "$(docker ps -a |grep ${DOCKER_BUILD_ENV_NAME})"; then
$docker_cmd run -it -v ${MOUNT_PROJECT_HOME}:/code/Turing/graphEngine --workdir ${docker_work_dir} --name ${DOCKER_BUILD_ENV_NAME} ${DOCKER_FULL_IMAGE_NAME} ${docker_bash_dir}
elif test -z "$(docker ps |grep ${DOCKER_BUILD_ENV_NAME})"; then
$docker_cmd start ${DOCKER_BUILD_ENV_NAME}
$docker_cmd exec -w ${docker_work_dir} -it ${DOCKER_BUILD_ENV_NAME} ${docker_bash_dir}
else
$docker_cmd exec -w ${docker_work_dir} -it ${DOCKER_BUILD_ENV_NAME} ${docker_bash_dir}
fi
}

function resert_docker_env(){
if test -z "$(docker ps -a |grep ${DOCKER_BUILD_ENV_NAME})"; then
echo "no runing container for graphengine build"
elif test -z "$(docker ps |grep ${DOCKER_BUILD_ENV_NAME})"; then
$docker_cmd rm ${DOCKER_BUILD_ENV_NAME}
else
$docker_cmd stop ${DOCKER_BUILD_ENV_NAME}
$docker_cmd rm ${DOCKER_BUILD_ENV_NAME}
fi
}

function help(){
cat <<-EOF
Usage: ge env [OPTIONS]

Prepare for docker env for build and test

Options:
-b, --build Build docker image
-p, --pull Pull docker image
-e, --enter Enter container
-r, --reset Reset container
-h, --help
EOF

}

function parse_args(){
parsed_args=$(getopt -a -o bperh --long build,pull,enter,resethelp -- "$@") || {
help
exit 1
}

if [ $# -lt 1 ]; then
pull_docker_image
enter_docker_env
exit 1
fi

eval set -- "$parsed_args"
while true; do
case "$1" in
-b | --build)
build_docker_image
;;
-p | --pull)
pull_docker_image
;;
-e | --enter)
enter_docker_env
;;
-r | --reset)
resert_docker_env
;;
-h | --help)
help
;;
--)
shift; break;
;;
*)
help; exit 1
;;
esac
shift
done
}

function main(){
parse_args "$@"
}

main "$@"
set -e

scripts/format_source_code.sh → scripts/format/ge_format.sh View File

@@ -1,5 +1,5 @@
#!/bin/bash
# Copyright 2019-2020 Huawei Technologies Co., Ltd
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -24,11 +24,12 @@ if [[ "${version}" -lt "8" ]]; then
exit 1
fi


CURRENT_PATH=$(pwd)
SCRIPTS_PATH=$(dirname "$0")
PROJECT_HOME=${PROJECT_HOME:-$(dirname "$0")/../../}

echo "CURRENT_PATH=${CURRENT_PATH}"
echo "SCRIPTS_PATH=${SCRIPTS_PATH}"
echo "PROJECT_HOME=${PROJECT_HOME}"

# print usage message
function usage()
@@ -81,27 +82,28 @@ function checkopts()
checkopts "$@"

# switch to project root path, which contains clang-format config file '.clang-format'
cd "${SCRIPTS_PATH}/.." || exit 1

FMT_FILE_LIST='__format_files_list__'

if [[ "X${mode}" == "Xall" ]]; then
find src -type f -name "*" | grep "\.h$\|\.cc$" > "${FMT_FILE_LIST}" || true
find inc -type f -name "*" | grep "\.h$\|\.cc$" >> "${FMT_FILE_LIST}" || true
elif [[ "X${mode}" == "Xchanged" ]]; then
# --diff-filter=ACMRTUXB will ignore deleted files in commit
git diff --diff-filter=ACMRTUXB --name-only | grep "^inc\|^src" | grep "\.h$\|\.cc$" >> "${FMT_FILE_LIST}" || true
else # "X${mode}" == "Xlastcommit"
git diff --diff-filter=ACMRTUXB --name-only HEAD~ HEAD | grep "^inc\|^src" | grep "\.h$\|\.cc$" > "${FMT_FILE_LIST}" || true
fi

while read line; do
if [ -f "${line}" ]; then
${CLANG_FORMAT} -i "${line}"
fi
done < "${FMT_FILE_LIST}"
pushd "${CURRENT_PATH}"
cd "${PROJECT_HOME}" || exit 1
FMT_FILE_LIST='__format_files_list__'

if [[ "X${mode}" == "Xall" ]]; then
find src -type f -name "*" | grep "\.h$\|\.cc$" > "${FMT_FILE_LIST}" || true
find inc -type f -name "*" | grep "\.h$\|\.cc$" >> "${FMT_FILE_LIST}" || true
elif [[ "X${mode}" == "Xchanged" ]]; then
# --diff-filter=ACMRTUXB will ignore deleted files in commit
git diff --diff-filter=ACMRTUXB --name-only | grep "^inc\|^src" | grep "\.h$\|\.cc$" >> "${FMT_FILE_LIST}" || true
else # "X${mode}" == "Xlastcommit"
git diff --diff-filter=ACMRTUXB --name-only HEAD~ HEAD | grep "^inc\|^src" | grep "\.h$\|\.cc$" > "${FMT_FILE_LIST}" || true
fi

while read line; do
if [ -f "${line}" ]; then
${CLANG_FORMAT} -i "${line}"
fi
done < "${FMT_FILE_LIST}"

rm "${FMT_FILE_LIST}"
cd "${CURRENT_PATH}" || exit 1
rm "${FMT_FILE_LIST}"
popd

echo "Specified cpp source files have been format successfully."

+ 77
- 0
scripts/ge.sh View File

@@ -0,0 +1,77 @@
#!/bin/bash
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================

GE_BASH_HOME=$(dirname "$0")
export PROJECT_HOME=${PROJECT_HOME:-${GE_BASH_HOME}/../}
PROJECT_HOME=$(cd $PROJECT_HOME || return; pwd)

function help(){
cat <<-EOF
Usage: ge COMMANDS

Run ge commands

Commands:
env Prepare docker env
config Config dependencies server
update Update dependencies
format Format code
build Build code
test Run test of UT/ST
cov Run Coverage
docs Generate documents
clean Clean
EOF

}

function ge_error() {
echo "Error: $*" >&2
help
exit 1
}

function main(){
if [ $# -eq 0 ]; then
help; exit 0
fi

local cmd=$1
local shell_cmd=
shift

case "$cmd" in
-h|--help)
help; exit 0
;;
build)
shell_cmd=${PROJECT_HOME}/build.sh
;;
*)
shell_cmd=$GE_BASH_HOME/$cmd/ge_$cmd.sh
;;

esac

[ -e $shell_cmd ] || {
ge_error "ge $shell_cmd is not found"
}

$shell_cmd "$@"
}

main "$@"


+ 331
- 0
scripts/readme.md View File

@@ -0,0 +1,331 @@
# graph engine 个人开发工具链使用说明

GE开发者工具链是graph engine中的一套面向个人开发者的自动化脚本工具链。

目前支持基于容器开发环境准备、构建依赖的自动下载安装和配置、代码格式化、编译、测试、代码覆盖率检查、文档生成等一系列开发者常用功能。

## 前置准备

下面是使用GE开发者工具链，需要手动进行的前置准备；

下列是经过验证后的性能最佳推荐配置：

1. 操作系统，以下任选其一：
- 原生的Linux操作系统，如ubuntu；
- Windows操作系统，推荐安装WSL的ubuntu系统，强烈建议登录WSL内直接下载代码，不要挂卷（构建性能差）！
- MAC OS；

2. docker安装：
- docker安装成功，并且相关镜像源已经设置正确，可正常下载外部镜像。

3. OS支持的命令行工具： 原生Linux or WSL shell；

可用但不推荐的配置：

- 在windows中直接安装docker，采用仿linux bash（Cygwin，minGW等）执行ge工具链；
（使用这种方式也可以执行所有GE工具链的操作，但是因为windows和容器异构内核的文件访问限制会导致构建速度比较慢）

## 快速上手

GE工具链对应的脚本在scripts下，可以按照下面流程来执行：

1. 进入到scripts目录：

```sh
$ cd ./scripts
```

2.执行`ge env`自动下载容器环境，并登陆到环境中

```sh
$ ./ge.sh env
```

3.配置外部依赖服务器信息
```sh
ge config -i=121.36.**.** -u=asc**, -p=Asc***\#@\!$ (Need add escape character \ before special charactor $、#、!)
```

4.下载和安装构建所依赖的外部库

```sh
$ ge update
```

（注：进入容器后，`ge`命令已经自动注册进系统，因此容器内不需要写脚本全称）

5.执行测试，默认执行单元测试用例，`ge test`会自动触发构建

```sh
$ ge test
```

## 详细用法

在scripts目录下，运行./ge.sh -h 即可查看到所有的子命令集合。

```sh
$ ./ge.sh -h

Usage: ge COMMANDS

Run ge commands

Commands:
env Prepare docker env
config Config dependencies server
update Update dependencies
format Format code
lint Static verify
build Build code
test Run test of UT/ST
cov Run Coverage
docs Generate documents
clean Clean
```

脚本内置的每个子命令，代表一个独立功能；每个子命令还提供了二级参数用于灵活指定执行方式。

每个子命令可以通过`-h`查看支持的可配参数。

例如查询`env`子命令的参数，可以使用如下命令：

```sh
$ ./ge.sh env -h
```

每个子命令在不带参数时，会有一个默认的行为。

### `ge env`

该命令用于准备构建和测试使用的容器环境，具体包含参数如下：

```
$ ./ge.sh env -h

Usage: ge env [OPTIONS]

Prepare for docker env for build and test

Options:
-b, --build Build docker image
-p, --pull Pull docker image
-e, --enter Enter container
-r, --reset Reset container
-h, --help
```

参数详细解释：
- `-b -- build`： 依据“scripts/env/Dockerfile”生成需要运行的容器镜像；
- `-p -- pull` ： 从本地配置的容器中央仓拉取需要的的容器镜像；
- `-e -- enter`： 在本地已有容器镜像的前提下，登录容器运行环境；
- `-r -- reset`： 删除本地运行的容器镜像环境；

默认：从中央容器仓拉取对应的容器镜像，运行实例并登陆。

### `ge config`

配置外部依赖服务器，具体参数如下：

```sh
$ ge config -h

Usage: ge config [OPTIONS]

update server config for ge, you need input all config info (ip, user, password)

Options:
-i, --ip Config ip config
-u, --user Config user name
-p, --password Config password
-h, --help

Example: ge config -i=121.36.**.** -u=asc**, -p=Asc***\#@\!$ (Need add escape character \ before special charactor $、#、!)
```

参数详细解释：

- `-i, --ip` : 配置依赖库服务器IP地址；
- `-u, --usr` : 配置依赖库服务器用户名；
- `-p, --password` : 配置依赖库地址；

默认：打印帮助信息。


### `ge update`

安装graph engine构建所需的外部依赖库，具体参数如下：

```sh
$ ge update -h

Usage: ge update [OPTIONS]

update dependencies of build and test

Options:
-d, --download Download dependencies
-i, --install Install dependencies
-c, --clear Clear dependencies
-h, --help
```

参数详细解释：

- `-d, --download` : 下载构建需要外部依赖库；
- `-i, --install` : 安装外部依赖包到对应位置；
- `-c, --clear` : 清除下载的外部依赖包；

默认：根据"scripts/update/deps_config.sh"的配置下载外部依赖库并安装到对应目录。
（注：请确保“scripts/update/server_config.sh”中的服务器地址、用户名、密码已经配置）

### `ge format`

使用clang-format进行代码格式化，具体参数如下：

```sh
$ ge format -h
Options:
-a format of all files
-c format of the files changed compared to last commit, default case
-l format of the files changed in last commit
-h Print usage
```

参数详细解释：

- `-a` : 格式化所有代码；
- `-c` : 只格式化本次修改的代码；
- `-l` : 格式化上次提交的代码；

默认：格式化本次修改代码。

### `ge lint`

使用clang-format进行代码格式化检查，具体参数如下：

```sh
$ ge lint -h

Options:
-a Check code format of all files, default case
-c Check code format of the files changed compared to last commit
-l Check code format of the files changed in last commit
-h Print usage
```

参数详细解释：

- `-a` : 检查所有代码格式；
- `-c` : 只检查修改的代码格式；
- `-l` : 检查上次提交的代码格式；

默认：检查本次修改代码格式。

### `ge build`

执行构建 (注：调用原有build.sh脚本，改造中...)；

### `ge test`

构建和运行测试用例，目前可以支持参数如下：

```sh
$ ge test -h

Usage: ge test [OPTIONS]

Options:
-u, --unit Run unit Test
-c, --component Run component Test
-h, --help
```

参数详细解释：

- `-u, --unit` : 执行单元测试
- `-c, --component` : 执行组件测试

默认：执行单元测试。

### `ge cov`

执行代码覆盖率检查, 支持全量覆盖和增量覆盖的功能，该命令需要已经跑完测试用例，目前可以支持参数如下：

```sh
$ ge cov -h

Usage: ge cov [OPTIONS]

Options:
-a, --all Full coverage
-i, --increment Increment coverage
-d, --directory Coverage of directory
-h, --help
```

参数详细解释：

- `-a, --all` : 执行全量覆盖率统计；
- `-i, --increment` : 执行增量覆盖率检查，默认是分析未提交修改的代码覆盖率（如果存在新增加的git未跟踪文件，需要先git add 添加进来才可以）；
- `-d, --directory` : 代码进行增量覆盖率检查的代码路径，支持传入路径参数；

默认：执行增量覆盖率检查；

下面的命令演示了如何检查ge目录下所有代码的增量覆盖率：

```sh
$ ge cov -d=ge
```

### `ge docs`

Doxygen文档生成，包含代码逻辑和物理结构和关系，方便阅读和理解代码；目前可以支持参数如下：

```sh
$ ge docs -h

Usage: ge docs [OPTIONS]

Options:
-b, --brief Build brief docs
-a, --all Build all docs
-h, --help
```

参数详细解释：

- `-b, --brief` : 生成简要文档，忽略部分关系图生成，速度快；
- `-a, --all` : 生成全量文档，包含各种代码关系图，速度相对慢；

默认： 生成全量代码文档。

### `ge clean`

清除各种下载或生成的中间文件，目前支持的参数如下：

```sh
$ ge clean -h

Usage: ge clean [OPTIONS]

Options:
-b, --build Clean build dir
-d, --docs Clean generate docs
-i, --install Clean dependenices
-a, --all Clean all
-h, --help
```

参数详细解释：

- `-b, --build` : 清除生成的编译构建临时文件；
- `-d, --docs` : 清除生成的文档临时文件；
- `-i, --install` : 清除安装的依赖文件；
- `-a, --all` : 清除所有下载和生成的临时文件；

默认：清除编译构建产生临时文件。

## Follow us

工具链的功能还在不断完善中，有问题请提issue，谢谢！

+ 80
- 0
scripts/test/ge_test.sh View File

@@ -0,0 +1,80 @@
#!/bin/bash
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
set -e

PROJECT_HOME=${PROJECT_HOME:-$(dirname "$0")/../../}

function help(){
cat <<-EOF
Usage: ge test [OPTIONS]

Options:
-u, --unit Run unit Test
-c, --component Run component Test
-h, --help
EOF

}

function unit_test(){
${PROJECT_HOME}/build.sh -u
}

function component_test(){
${PROJECT_HOME}/build.sh -s
}

function parse_args(){
parsed_args=$(getopt -a -o uch --long unit,component,help -- "$@") || {
help
exit 1
}

if [ $# -lt 1 ]; then
unit_test
exit 1
fi

eval set -- "$parsed_args"
while true; do
case "$1" in
-u | --unit)
unit_test
;;
-c | --component)
component_test
;;
-h | --help)
help
;;
--)
shift; break;
;;
*)
help; exit 1
;;
esac
shift
done
}

function main(){
parse_args "$@"
}

main "$@"

set +e

+ 47
- 0
scripts/update/deps_config.sh View File

@@ -0,0 +1,47 @@
#!/bin/bash
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
set -e

SERVER_CONFIG_FILE=${PROJECT_HOME}/scripts/config/server_config.sh

[ -e $SERVER_CONFIG_FILE ] || {
echo "You have not config dependencies account info first !!!!!"
${PROJECT_HOME}/scripts/config/ge_config.sh -h
exit 1;
}

source scripts/config/server_config.sh

CPU_ARCH=ubuntu18.04.x86_64
DRIVER_VERSION=20.2.0
CHIP_NAME=A800-9010
PRODUCT_VERSION=driver_C76_TR5

DRIVER_NAME=npu-driver
DRIVER_RUN_NAME=${CHIP_NAME}-${DRIVER_NAME}_${DRIVER_VERSION}_ubuntu18.04-x86_64.run

DEV_TOOLS_VERSION=1.78.t10.0.b100

export ATC_RUN_NAME=Ascend-atc-${DEV_TOOLS_VERSION}-${CPU_ARCH}.run
export ACL_RUN_NAME=Ascend-acllib-${DEV_TOOLS_VERSION}-${CPU_ARCH}.run
export FWKACL_RUN_NAME=Ascend-fwkacllib-${DEV_TOOLS_VERSION}-${CPU_ARCH}.run

DEV_TOOLS_PACKAGE=x86_ubuntu_os_devtoolset_package

export DRIVER_URL=${SERVER_PATH}/${PRODUCT_VERSION}/${DRIVER_RUN_NAME}
export DEV_TOOLS_URL=${SERVER_PATH}/20210428/${DEV_TOOLS_PACKAGE}.zip

set +e

+ 136
- 0
scripts/update/ge_update.sh View File

@@ -0,0 +1,136 @@
#!/bin/bash
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
set -e

PROJECT_HOME=${PROJECT_HOME:-$(dirname "$0")/../../}
PROJECT_HOME=$(cd $PROJECT_HOME || return; pwd)
DOWNLOAD_PATH=${PROJECT_HOME}/deps
DEP_LIB_DIR=./lib
DEP_TMP_DIR=./tmp

function extract_deps_so()
{
echo "begin to extract .run file ........."
chmod 777 ./${DRIVER_RUN_NAME}
unzip ${DEV_TOOLS_PACKAGE}.zip
chmod -R 777 ${DEV_TOOLS_PACKAGE}
[ -n "${DEP_TMP_DIR}" ] && rm -rf "${DEP_TMP_DIR}"
./${DRIVER_RUN_NAME} --noexec --extract=${DEP_TMP_DIR}/driver
./${DEV_TOOLS_PACKAGE}/${ATC_RUN_NAME} --noexec --extract=${DEP_TMP_DIR}/atc
./${DEV_TOOLS_PACKAGE}/${ACL_RUN_NAME} --noexec --extract=${DEP_TMP_DIR}/acllib
./${DEV_TOOLS_PACKAGE}/${FWKACL_RUN_NAME} --noexec --extract=${DEP_TMP_DIR}/fwkacllib
}

function copy_so_to_target_dir()
{
mkdir -p $DEP_LIB_DIR
mv ${DEP_TMP_DIR}/driver/driver $DEP_LIB_DIR/driver
mv ${DEP_TMP_DIR}/atc/atc $DEP_LIB_DIR/atc
mv ${DEP_TMP_DIR}/acllib/acllib $DEP_LIB_DIR/acllib
mv ${DEP_TMP_DIR}/fwkacllib/fwkacllib $DEP_LIB_DIR/fwkacllib
}

function clear_libs()
{
[ -n "${DOWNLOAD_PATH}" ] && rm -rf "${DOWNLOAD_PATH}"
}

function download_runs()
{
source scripts/update/deps_config.sh
echo "begin to download .run file ........."
clear_libs
mkdir -p ./ ${DOWNLOAD_PATH}
pushd "${DOWNLOAD_PATH}" >/dev/null
cd ${DOWNLOAD_PATH}
wget --user=${DEP_USER} --password=${DEP_PASSWORD} ${DRIVER_URL}
wget --user=${DEP_USER} --password=${DEP_PASSWORD} ${DEV_TOOLS_URL}
popd >/dev/null

}

function install_deps()
{
source scripts/update/deps_config.sh
mkdir -p ./ ${DOWNLOAD_PATH}
pushd "${DOWNLOAD_PATH}" >/dev/null
cd ${DOWNLOAD_PATH}
extract_deps_so
copy_so_to_target_dir
popd >/dev/null
}


function help(){
cat <<-EOF
Usage: ge update [OPTIONS]

update dependencies of build and test

Options:
-d, --download Download dependencies
-i, --install Install dependencies
-c, --clear Clear dependencies
-h, --help
EOF

}

function parse_args(){
parsed_args=$(getopt -a -o dich --long download,install,clear,help -- "$@") || {
help
exit 1
}

if [ $# -lt 1 ]; then
download_runs
install_deps
exit 1
fi

eval set -- "$parsed_args"
while true; do
case "$1" in
-d | --download)
download_runs
;;
-i | --install)
install_deps
;;
-c | --clear)
clear_libs
;;
-h | --help)
help; exit 1;
;;
--)
shift; break;
;;
*)
help; exit 1
;;
esac
shift
done
}

function main(){
parse_args "$@"
}

main "$@"

set +e

Write Preview
Loading…
Cancel
Save