hummingbird
/
graphengine
Not watched
2
0
Fork 0
Code
Releases 13 Wiki Activity
Issues 0 Pull Requests 0 Datasets Model Cloudbrain
Browse Source

!1686 Add a tool chain for Developer for automated build environment installation, dependent installation, etc. 

From: @wei-gangqiang
Reviewed-by: @sheng-nan
Signed-off-by:
tags/v1.3.0
mindspore-ci-bot Gitee 4 years ago
parent
f04ec9c790 f5dc59ad7d
commit
10f952b818
18 changed files with 6661 additions and 61 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +9
    -0
      .gitignore
  2. +0
    -0
      build.sh
  3. +25
    -0
      scripts/build/ge_build.sh
  4. +38
    -38
      scripts/check_clang_format.sh
  5. +90
    -0
      scripts/clean/ge_clean.sh
  6. +115
    -0
      scripts/config/ge_config.sh
  7. +136
    -0
      scripts/cov/ge_cov.sh
  8. +2639
    -0
      scripts/docs/Doxyfile_all
  9. +2638
    -0
      scripts/docs/Doxyfile_brief
  10. +87
    -0
      scripts/docs/ge_docs.sh
  11. +42
    -0
      scripts/env/Dockerfile
  12. +146
    -0
      scripts/env/ge_env.sh
  13. +25
    -23
      scripts/format/ge_format.sh
  14. +77
    -0
      scripts/ge.sh
  15. +331
    -0
      scripts/readme.md
  16. +80
    -0
      scripts/test/ge_test.sh
  17. +47
    -0
      scripts/update/deps_config.sh
  18. +136
    -0
      scripts/update/ge_update.sh

+ 9
- 0
.gitignore View File

@@ -3,8 +3,17 @@
/output
/prebuilts
/cov
/deps
.autotools
.project
.cproject
.settings/
/tests/frm/
*.ir
*.out
*.DS_Store
.DS_Store
server_config.sh

# Dynamic libraries
# *.so


+ 0
- 0
build.sh View File


+ 25
- 0
scripts/build/ge_build.sh View File

@@ -0,0 +1,25 @@
#!/bin/bash
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================

set -e
export PROJECT_HOME=${PROJECT_HOME:-$(dirname "$0")/../}
function main(){
${PROJECT_HOME}/build.sh "$@"
}

main "$@"

set +e

+ 38
- 38
scripts/check_clang_format.sh View File

@@ -1,5 +1,5 @@
#!/bin/bash
# Copyright 2019-2020 Huawei Technologies Co., Ltd
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -13,7 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================

set -e

CLANG_FORMAT=$(which clang-format) || (echo "Please install 'clang-format' tool first"; exit 1)
@@ -25,10 +24,10 @@ if [[ "${version}" -lt "8" ]]; then
fi

CURRENT_PATH=$(pwd)
SCRIPTS_PATH=$(dirname "$0")
PROJECT_HOME=${PROJECT_HOME:-$(dirname "$0")/../}

echo "CURRENT_PATH=$CURRENT_PATH"
echo "SCRIPTS_PATH=$SCRIPTS_PATH"
echo "PROJECT_HOME=$PROJECT_HOME"

# print usage message
function usage()
@@ -81,45 +80,46 @@ function checkopts()
checkopts "$@"

# switch to project root path, which contains clang-format config file '.clang-format'
cd "${SCRIPTS_PATH}/.." || exit 1
pushd "${CURRENT_PATH}"
CHECK_LIST_FILE='__checked_files_list__'
cd "${PROJECT_HOME}" || exit 1

CHECK_LIST_FILE='__checked_files_list__'
if [ "X${mode}" == "Xall" ]; then
find src -type f -name "*" | grep "\.h$\|\.cc$" > "${CHECK_LIST_FILE}" || true
find inc -type f -name "*" | grep "\.h$\|\.cc$" >> "${CHECK_LIST_FILE}" || true
elif [ "X${mode}" == "Xchanged" ]; then
# --diff-filter=ACMRTUXB will ignore deleted files in commit
git diff --diff-filter=ACMRTUXB --name-only | grep "^inc\|^src" | grep "\.h$\|\.cc$" > "${CHECK_LIST_FILE}" || true
else # "X${mode}" == "Xlastcommit"
git diff --diff-filter=ACMRTUXB --name-only HEAD~ HEAD | grep "^inc\|^src" | grep "\.h$\|\.cc$" > "${CHECK_LIST_FILE}" || true
fi

if [ "X${mode}" == "Xall" ]; then
find src -type f -name "*" | grep "\.h$\|\.cc$" > "${CHECK_LIST_FILE}" || true
find inc -type f -name "*" | grep "\.h$\|\.cc$" >> "${CHECK_LIST_FILE}" || true
elif [ "X${mode}" == "Xchanged" ]; then
# --diff-filter=ACMRTUXB will ignore deleted files in commit
git diff --diff-filter=ACMRTUXB --name-only | grep "^inc\|^src" | grep "\.h$\|\.cc$" > "${CHECK_LIST_FILE}" || true
else # "X${mode}" == "Xlastcommit"
git diff --diff-filter=ACMRTUXB --name-only HEAD~ HEAD | grep "^inc\|^src" | grep "\.h$\|\.cc$" > "${CHECK_LIST_FILE}" || true
fi
CHECK_RESULT_FILE=__code_format_check_result__
echo "0" > "$CHECK_RESULT_FILE"

CHECK_RESULT_FILE=__code_format_check_result__
echo "0" > "$CHECK_RESULT_FILE"
# check format of files modified in the lastest commit
while read line; do
BASE_NAME=$(basename "${line}")
TEMP_FILE="__TEMP__${BASE_NAME}"
cp "${line}" "${TEMP_FILE}"
${CLANG_FORMAT} -i "${TEMP_FILE}"
set +e
diff "${TEMP_FILE}" "${line}"
ret=$?
set -e
rm "${TEMP_FILE}"
if [[ "${ret}" -ne 0 ]]; then
echo "File ${line} is not formated, please format it."
echo "1" > "${CHECK_RESULT_FILE}"
break
fi
done < "${CHECK_LIST_FILE}"

# check format of files modified in the lastest commit
while read line; do
BASE_NAME=$(basename "${line}")
TEMP_FILE="__TEMP__${BASE_NAME}"
cp "${line}" "${TEMP_FILE}"
${CLANG_FORMAT} -i "${TEMP_FILE}"
set +e
diff "${TEMP_FILE}" "${line}"
ret=$?
set -e
rm "${TEMP_FILE}"
if [[ "${ret}" -ne 0 ]]; then
echo "File ${line} is not formated, please format it."
echo "1" > "${CHECK_RESULT_FILE}"
break
fi
done < "${CHECK_LIST_FILE}"
result=$(cat "${CHECK_RESULT_FILE}")
rm "${CHECK_RESULT_FILE}"
rm "${CHECK_LIST_FILE}"
popd

result=$(cat "${CHECK_RESULT_FILE}")
rm "${CHECK_RESULT_FILE}"
rm "${CHECK_LIST_FILE}"
cd "${CURRENT_PATH}" || exit 1
if [[ "X${result}" == "X0" ]]; then
echo "Check PASS: specified files are well formated!"
fi


+ 90
- 0
scripts/clean/ge_clean.sh View File

@@ -0,0 +1,90 @@
#!/bin/bash
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================

set -e

PROJECT_HOME=${PROJECT_HOME:-$(dirname "$0")/../../}

function help(){
cat <<-EOF
Usage: ge clean [OPTIONS]

Options:
-b, --build Clean build dir
-d, --docs Clean generate docs
-i, --install Clean dependenices
-a, --all Clean all
-h, --help
EOF

}

function clean_relative_dir(){
rm -rf "${PROJECT_HOME}/${1:-output}"
}

function parse_args(){
parsed_args=$(getopt -a -o bdiah --long build,docs,install,all,help -- "$@") || {
help
exit 1
}

if [ $# -lt 1 ]; then
clean_relative_dir "build"
clean_relative_dir "output"
exit 1
fi

eval set -- "$parsed_args"
while true; do
case "$1" in
-b | --build)
clean_relative_dir "build"
clean_relative_dir "output"
;;
-d | --docs)
clean_relative_dir "docs/doxygen"
;;
-i | --install)
clean_relative_dir "deps"
;;
-a | --all)
clean_relative_dir "deps"
clean_relative_dir "build"
clean_relative_dir "output"
clean_relative_dir "docs/doxygen"
;;
-h | --help)
help
;;
--)
shift; break;
;;
*)
help; exit 1
;;
esac
shift
done
}

function main(){
parse_args "$@"
}

main "$@"

set +e

+ 115
- 0
scripts/config/ge_config.sh View File

@@ -0,0 +1,115 @@
#!/bin/bash
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================

set -e

PROJECT_HOME=${PROJECT_HOME:-$(dirname "$0")/../../}
PROJECT_HOME=$(cd $PROJECT_HOME || return; pwd)

function help(){
cat <<-EOF
Usage: ge config [OPTIONS]

update server config for ge, you need input all config info (ip, user, password)

Options:
-i, --ip Config ip config
-u, --user Config user name
-p, --password Config password
-h, --help

Example: ge config -i=121.36.**.** -u=asc**, -p=Asc***\#@\!\$ (Need add escape character \ before special charactor $、#、!)

EOF

}

function write_config_file(){
local IP=$1
local USER=$2
local PASSWORD=$3
if [[ -z "$IP" ]] || [[ -z "$USER" ]] || [[ -z "$USER" ]]; then
echo "You need input all info （ip, user,password）obout server config !!!"
help
exit 1
fi

local PASSWORD=${PASSWORD//!/\\!}
local PASSWORD=${PASSWORD//#/\\#}
local PASSWORD=${PASSWORD/\$/\\\$}
local SERVER_CONFIG_FILE=${PROJECT_HOME}/scripts/config/server_config.sh
[ -n "${SERVER_CONFIG_FILE}" ] && rm -rf "${SERVER_CONFIG_FILE}"

cat>${SERVER_CONFIG_FILE}<<-EOF
SERVER_PATH=http://${IP}/package/etrans
DEP_USER=${USER}
DEP_PASSWORD=${PASSWORD}

EOF

}



function parse_args(){
parsed_args=$(getopt -a -o i::u::p::h --long ip::,user::,password::,help -- "$@") || {
help
exit 1
}

if [ $# -lt 1 ]; then
help
exit 1
fi
local IP=
local USER=
local PASSWORD=

eval set -- "$parsed_args"
while true; do
case "$1" in
-i | --ip)
IP=$2
;;
-u | --user)
USER=$2
;;
-p | --password)
PASSWORD=$2
;;
-h | --help)
help; exit;
;;
--)
shift; break;
;;
*)
help; exit 1
;;
esac
shift 2
done

write_config_file $IP $USER $PASSWORD
}

function main(){
parse_args "$@"
}

main "$@"

set +e

+ 136
- 0
scripts/cov/ge_cov.sh View File

@@ -0,0 +1,136 @@
#!/bin/bash
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================

set -e


function help(){
cat <<-EOF
Usage: ge cov [OPTIONS]

Options:
-a, --all Full coverage
-i, --increment Increment coverage
-d, --directory Coverage of directory
-h, --help
EOF

}

PROJECT_HOME=${PROJECT_HOME:-$(dirname "$0")/../../}
PROJECT_HOME=$(cd $PROJECT_HOME || return; pwd)

ALL_COV_GEN_PATH=${PROJECT_HOME}/cov/all
DIFF_FILE_PATH=${PROJECT_HOME}/cov/diff
DIFF_FILE_NAME=${DIFF_FILE_PATH}/inc_change_diff.txt

function process_diff_format(){
sed -i "s/--- a/--- \/code\/Turing\/graphEngine/g" ${DIFF_FILE_NAME}
sed -i "s/+++ b/+++ \/code\/Turing\/graphEngine/g" ${DIFF_FILE_NAME}
}


function add_cov_generate(){
addlcov --diff ${ALL_COV_GEN_PATH}/coverage.info ${DIFF_FILE_NAME} -o ${PROJECT_HOME}/cov/diff/inc_coverage.info
}

function gen_add_cov_html(){
genhtml --prefix ${PROJECT_HOME} -o ${PROJECT_HOME}/cov/diff/html ${PROJECT_HOME}/cov/diff/inc_coverage.info --legend -t CHG --no-branch-coverage --no-function-coverage
}

function increment_cov_for_directory(){
[ -n "${DIFF_FILE_PATH}" ] && rm -rf "${DIFF_FILE_PATH}"
mkdir -p ${DIFF_FILE_PATH}
git diff HEAD -- $1 >>${DIFF_FILE_NAME}
process_diff_format
add_cov_generate
gen_add_cov_html
}

function run_all_coverage(){
[ -n "${ALL_COV_GEN_PATH}" ] && rm -rf ${ALL_COV_GEN_PATH}
mkdir -p ${ALL_COV_GEN_PATH}
pushd "${PWD}" >/dev/null
cd ${PROJECT_HOME}
lcov -c -d build/tests/ut/ge -d build/tests/ut/common/graph/ -o ${ALL_COV_GEN_PATH}/tmp.info
lcov -r ${ALL_COV_GEN_PATH}/tmp.info '*/output/*' '*/build/opensrc/*' '*/build/proto/*' '*/third_party/*' '*/tests/*' '/usr/local/*' '/usr/include/*' '*/metadef/*' '*/parser/*' -o ${ALL_COV_GEN_PATH}/coverage.info
cd ${ALL_COV_GEN_PATH}
genhtml coverage.info
popd >/dev/null
}

function do_coverage_run(){
local cov_mode=$1
local directory_dir=$2
run_all_coverage

if [ "$cov_mode" = "all" ]; then
exit 1
elif [ -n "$directory_dir" ]; then
increment_cov_for_directory $directory_dir
else
increment_cov_for_directory "ge"
fi
}

function parse_args(){
parsed_args=$(getopt -a -o aid::h --long all,increment,directory::,help -- "$@") || {
help
exit 1
}

if [ $# -lt 1 ]; then
run_all_coverage
exit 1
fi

local cov_mode="increment"
local directory_dir=
eval set -- "$parsed_args"
while true; do
case "$1" in
-a | --all)
cov_mode="all"
;;
-i | --increment)
;;
-d | --directory)
directory_dir=$2
shift
;;
-h | --help)
help; exit 1;
;;
--)
shift; break;
;;
*)
help; exit 1;
;;
esac
shift
done
do_coverage_run $cov_mode $directory_dir
}

function main(){
parse_args "$@"
}

main "$@"

set +e

+ 2639
- 0
scripts/docs/Doxyfile_all
File diff suppressed because it is too large
View File


+ 2638
- 0
scripts/docs/Doxyfile_brief
File diff suppressed because it is too large
View File


+ 87
- 0
scripts/docs/ge_docs.sh View File

@@ -0,0 +1,87 @@
#!/bin/bash
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================

set -e

function help(){
cat <<-EOF
Usage: ge docs [OPTIONS]

Options:
-b, --brief Build brief docs
-a, --all Build all docs
-h, --help
EOF

}

PROJECT_HOME=${PROJECT_HOME:-$(dirname "$0")/../../}
PROJECT_HOME=$(cd $PROJECT_HOME || return; pwd)
BRIEF_DOXYFILE_PATH=${PROJECT_HOME}/scripts/docs/Doxyfile_brief
ALL_DOXYFILE_PATH=${PROJECT_HOME}/scripts/docs/Doxyfile_all


function build_brief_docs(){
rm -rf "${PROJECT_HOME}/docs/doxygen"
doxygen ${BRIEF_DOXYFILE_PATH}
}

function build_all_docs(){
rm -rf "${PROJECT_HOME}/docs/doxygen"
doxygen ${ALL_DOXYFILE_PATH}
}

function parse_args(){
parsed_args=$(getopt -a -o bah --long brief,all,help -- "$@") || {
help
exit 1
}

if [ $# -lt 1 ]; then
build_all_docs
exit 1
fi

eval set -- "$parsed_args"
while true; do
case "$1" in
-b | --brief)
build_brief_docs
;;
-a | --all)
build_all_docs
;;
-h | --help)
help; exit 1;
;;
--)
shift; break;
;;
*)
help; exit 1;
;;
esac
shift
done
}

function main(){
parse_args "$@"
}

main "$@"

set +e

+ 42
- 0
scripts/env/Dockerfile View File

@@ -0,0 +1,42 @@
# this dockerfile used for graphengine build
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================

FROM ubuntu:18.04
RUN apt-get update \
&& apt-get install -y git g++ wget unzip clang-format-9 build-essential lcov vim

# install for doxygen
RUN apt-get install -y graphviz doxygen

# install for graph ensy engine
RUN cpan install -y Graph::Easy

RUN wget https://cmake.org/files/v3.16/cmake-3.16.7-Linux-x86_64.tar.gz

RUN mkdir -p /opt/cmake-3.16.7 \
&& tar -xvf cmake-3.16.7-Linux-x86_64.tar.gz -C /opt/cmake-3.16.7 --strip-components=1 \
&& ln -sf /opt/cmake-3.16.7/bin/* /usr/bin/ \
&& mv /usr/bin/clang-format-9 /usr/bin/clang-format

RUN wget https://github.com/ccup/lcov/archive/refs/tags/add_lcov.tar.gz -O add_lcov.tar.gz \
&& mkdir -p /opt/addlcov1.0.0 \
&& tar -xvf add_lcov.tar.gz -C /opt/addlcov1.0.0 \
&& mv /opt/addlcov1.0.0/lcov-add_lcov/bin/lcov /usr/bin/addlcov

ENV PROJECT_HOME=/code/Turing/graphEngine

RUN echo "alias ge=/code/Turing/graphEngine/scripts/ge.sh">>~/.bashrc


+ 146
- 0
scripts/env/ge_env.sh View File

@@ -0,0 +1,146 @@
#!/bin/bash
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================

set -e

PROJECT_HOME=${PROJECT_HOME:-$(dirname "$0")/../../}
MOUNT_PROJECT_HOME=$(cd $PROJECT_HOME || return; pwd)

DOCKER_BUILD_ENV_NAME=${MOUNT_PROJECT_HOME#*/}
DOCKER_BUILD_ENV_NAME=${DOCKER_BUILD_ENV_NAME//\//\_}
DOCKER_IMAGE_TAG=ge_build_env.1.0.6
DOCKER_IAMGE_NAME=joycode2art/turing
DOCKER_FULL_IMAGE_NAME=${DOCKER_IAMGE_NAME}:${DOCKER_IMAGE_TAG}

if [ "$(uname)" == "Darwin" ]; then
#running on Mac OS
docker_cmd=docker
MOUNT_PROJECT_HOME=${MOUNT_PROJECT_HOME}
docker_work_dir=/code/Turing/graphEngine
docker_bash_dir=/bin/bash
elif [ "$(expr substr "$(uname -s)" 1 10)" == "MINGW32_NT" ] || [ "$(expr substr "$(uname -s)" 1 10)" == "MINGW64_NT" ]; then
#running on Windows
docker_cmd="winpty docker"
MOUNT_PROJECT_HOME=/${MOUNT_PROJECT_HOME}
docker_work_dir=//code/Turing/graphEngine
docker_bash_dir=//bin/bash
elif [ "$(expr substr "$(uname -s)" 1 5)" == "Linux" ]; then
#running on Linux
docker_cmd=docker
MOUNT_PROJECT_HOME=${PROJECT_HOME}
docker_work_dir=/code/Turing/graphEngine
docker_bash_dir=/bin/bash
fi

function build_docker_image(){
if test -z "$(docker images |grep ${DOCKER_IAMGE_NAME} | grep ${DOCKER_IMAGE_TAG})"; then
$docker_cmd build -t ${DOCKER_FULL_IMAGE_NAME} ${PROJECT_HOME}/scripts/env
else
echo "docker image for graph engine build is build ok...."
fi
}

function pull_docker_image(){
$docker_cmd pull $DOCKER_FULL_IMAGE_NAME
}

function enter_docker_env(){
if test -z "$(docker images |grep ${DOCKER_IAMGE_NAME} | grep ${DOCKER_IMAGE_TAG})"; then
echo "please run 'ge env --pull' to download images first!"
elif test -z "$(docker ps -a |grep ${DOCKER_BUILD_ENV_NAME})"; then
$docker_cmd run -it -v ${MOUNT_PROJECT_HOME}:/code/Turing/graphEngine --workdir ${docker_work_dir} --name ${DOCKER_BUILD_ENV_NAME} ${DOCKER_FULL_IMAGE_NAME} ${docker_bash_dir}
elif test -z "$(docker ps |grep ${DOCKER_BUILD_ENV_NAME})"; then
$docker_cmd start ${DOCKER_BUILD_ENV_NAME}
$docker_cmd exec -w ${docker_work_dir} -it ${DOCKER_BUILD_ENV_NAME} ${docker_bash_dir}
else
$docker_cmd exec -w ${docker_work_dir} -it ${DOCKER_BUILD_ENV_NAME} ${docker_bash_dir}
fi
}

function resert_docker_env(){
if test -z "$(docker ps -a |grep ${DOCKER_BUILD_ENV_NAME})"; then
echo "no runing container for graphengine build"
elif test -z "$(docker ps |grep ${DOCKER_BUILD_ENV_NAME})"; then
$docker_cmd rm ${DOCKER_BUILD_ENV_NAME}
else
$docker_cmd stop ${DOCKER_BUILD_ENV_NAME}
$docker_cmd rm ${DOCKER_BUILD_ENV_NAME}
fi
}

function help(){
cat <<-EOF
Usage: ge env [OPTIONS]

Prepare for docker env for build and test

Options:
-b, --build Build docker image
-p, --pull Pull docker image
-e, --enter Enter container
-r, --reset Reset container
-h, --help
EOF

}

function parse_args(){
parsed_args=$(getopt -a -o bperh --long build,pull,enter,resethelp -- "$@") || {
help
exit 1
}

if [ $# -lt 1 ]; then
pull_docker_image
enter_docker_env
exit 1
fi

eval set -- "$parsed_args"
while true; do
case "$1" in
-b | --build)
build_docker_image
;;
-p | --pull)
pull_docker_image
;;
-e | --enter)
enter_docker_env
;;
-r | --reset)
resert_docker_env
;;
-h | --help)
help
;;
--)
shift; break;
;;
*)
help; exit 1
;;
esac
shift
done
}

function main(){
parse_args "$@"
}

main "$@"
set -e

scripts/format_source_code.sh → scripts/format/ge_format.sh View File

@@ -1,5 +1,5 @@
#!/bin/bash
# Copyright 2019-2020 Huawei Technologies Co., Ltd
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -24,11 +24,12 @@ if [[ "${version}" -lt "8" ]]; then
exit 1
fi


CURRENT_PATH=$(pwd)
SCRIPTS_PATH=$(dirname "$0")
PROJECT_HOME=${PROJECT_HOME:-$(dirname "$0")/../../}

echo "CURRENT_PATH=${CURRENT_PATH}"
echo "SCRIPTS_PATH=${SCRIPTS_PATH}"
echo "PROJECT_HOME=${PROJECT_HOME}"

# print usage message
function usage()
@@ -81,27 +82,28 @@ function checkopts()
checkopts "$@"

# switch to project root path, which contains clang-format config file '.clang-format'
cd "${SCRIPTS_PATH}/.." || exit 1

FMT_FILE_LIST='__format_files_list__'

if [[ "X${mode}" == "Xall" ]]; then
find src -type f -name "*" | grep "\.h$\|\.cc$" > "${FMT_FILE_LIST}" || true
find inc -type f -name "*" | grep "\.h$\|\.cc$" >> "${FMT_FILE_LIST}" || true
elif [[ "X${mode}" == "Xchanged" ]]; then
# --diff-filter=ACMRTUXB will ignore deleted files in commit
git diff --diff-filter=ACMRTUXB --name-only | grep "^inc\|^src" | grep "\.h$\|\.cc$" >> "${FMT_FILE_LIST}" || true
else # "X${mode}" == "Xlastcommit"
git diff --diff-filter=ACMRTUXB --name-only HEAD~ HEAD | grep "^inc\|^src" | grep "\.h$\|\.cc$" > "${FMT_FILE_LIST}" || true
fi

while read line; do
if [ -f "${line}" ]; then
${CLANG_FORMAT} -i "${line}"
fi
done < "${FMT_FILE_LIST}"
pushd "${CURRENT_PATH}"
cd "${PROJECT_HOME}" || exit 1
FMT_FILE_LIST='__format_files_list__'

if [[ "X${mode}" == "Xall" ]]; then
find src -type f -name "*" | grep "\.h$\|\.cc$" > "${FMT_FILE_LIST}" || true
find inc -type f -name "*" | grep "\.h$\|\.cc$" >> "${FMT_FILE_LIST}" || true
elif [[ "X${mode}" == "Xchanged" ]]; then
# --diff-filter=ACMRTUXB will ignore deleted files in commit
git diff --diff-filter=ACMRTUXB --name-only | grep "^inc\|^src" | grep "\.h$\|\.cc$" >> "${FMT_FILE_LIST}" || true
else # "X${mode}" == "Xlastcommit"
git diff --diff-filter=ACMRTUXB --name-only HEAD~ HEAD | grep "^inc\|^src" | grep "\.h$\|\.cc$" > "${FMT_FILE_LIST}" || true
fi

while read line; do
if [ -f "${line}" ]; then
${CLANG_FORMAT} -i "${line}"
fi
done < "${FMT_FILE_LIST}"

rm "${FMT_FILE_LIST}"
cd "${CURRENT_PATH}" || exit 1
rm "${FMT_FILE_LIST}"
popd

echo "Specified cpp source files have been format successfully."

+ 77
- 0
scripts/ge.sh View File

@@ -0,0 +1,77 @@
#!/bin/bash
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================

GE_BASH_HOME=$(dirname "$0")
export PROJECT_HOME=${PROJECT_HOME:-${GE_BASH_HOME}/../}
PROJECT_HOME=$(cd $PROJECT_HOME || return; pwd)

function help(){
cat <<-EOF
Usage: ge COMMANDS

Run ge commands

Commands:
env Prepare docker env
config Config dependencies server
update Update dependencies
format Format code
build Build code
test Run test of UT/ST
cov Run Coverage
docs Generate documents
clean Clean
EOF

}

function ge_error() {
echo "Error: $*" >&2
help
exit 1
}

function main(){
if [ $# -eq 0 ]; then
help; exit 0
fi

local cmd=$1
local shell_cmd=
shift

case "$cmd" in
-h|--help)
help; exit 0
;;
build)
shell_cmd=${PROJECT_HOME}/build.sh
;;
*)
shell_cmd=$GE_BASH_HOME/$cmd/ge_$cmd.sh
;;

esac

[ -e $shell_cmd ] || {
ge_error "ge $shell_cmd is not found"
}

$shell_cmd "$@"
}

main "$@"


+ 331
- 0
scripts/readme.md View File

@@ -0,0 +1,331 @@
# graph engine 个人开发工具链使用说明

GE开发者工具链是graph engine中的一套面向个人开发者的自动化脚本工具链。

目前支持基于容器开发环境准备、构建依赖的自动下载安装和配置、代码格式化、编译、测试、代码覆盖率检查、文档生成等一系列开发者常用功能。

## 前置准备

下面是使用GE开发者工具链，需要手动进行的前置准备；

下列是经过验证后的性能最佳推荐配置：

1. 操作系统，以下任选其一：
- 原生的Linux操作系统，如ubuntu；
- Windows操作系统，推荐安装WSL的ubuntu系统，强烈建议登录WSL内直接下载代码，不要挂卷（构建性能差）！
- MAC OS；

2. docker安装：
- docker安装成功，并且相关镜像源已经设置正确，可正常下载外部镜像。

3. OS支持的命令行工具： 原生Linux or WSL shell；

可用但不推荐的配置：

- 在windows中直接安装docker，采用仿linux bash（Cygwin，minGW等）执行ge工具链；
（使用这种方式也可以执行所有GE工具链的操作，但是因为windows和容器异构内核的文件访问限制会导致构建速度比较慢）

## 快速上手

GE工具链对应的脚本在scripts下，可以按照下面流程来执行：

1. 进入到scripts目录：

```sh
$ cd ./scripts
```

2.执行`ge env`自动下载容器环境，并登陆到环境中

```sh
$ ./ge.sh env
```

3.配置外部依赖服务器信息
```sh
ge config -i=121.36.**.** -u=asc**, -p=Asc***\#@\!$ (Need add escape character \ before special charactor $、#、!)
```

4.下载和安装构建所依赖的外部库

```sh
$ ge update
```

（注：进入容器后，`ge`命令已经自动注册进系统，因此容器内不需要写脚本全称）

5.执行测试，默认执行单元测试用例，`ge test`会自动触发构建

```sh
$ ge test
```

## 详细用法

在scripts目录下，运行./ge.sh -h 即可查看到所有的子命令集合。

```sh
$ ./ge.sh -h

Usage: ge COMMANDS

Run ge commands

Commands:
env Prepare docker env
config Config dependencies server
update Update dependencies
format Format code
lint Static verify
build Build code
test Run test of UT/ST
cov Run Coverage
docs Generate documents
clean Clean
```

脚本内置的每个子命令，代表一个独立功能；每个子命令还提供了二级参数用于灵活指定执行方式。

每个子命令可以通过`-h`查看支持的可配参数。

例如查询`env`子命令的参数，可以使用如下命令：

```sh
$ ./ge.sh env -h
```

每个子命令在不带参数时，会有一个默认的行为。

### `ge env`

该命令用于准备构建和测试使用的容器环境，具体包含参数如下：

```
$ ./ge.sh env -h

Usage: ge env [OPTIONS]

Prepare for docker env for build and test

Options:
-b, --build Build docker image
-p, --pull Pull docker image
-e, --enter Enter container
-r, --reset Reset container
-h, --help
```

参数详细解释：
- `-b -- build`： 依据“scripts/env/Dockerfile”生成需要运行的容器镜像；
- `-p -- pull` ： 从本地配置的容器中央仓拉取需要的的容器镜像；
- `-e -- enter`： 在本地已有容器镜像的前提下，登录容器运行环境；
- `-r -- reset`： 删除本地运行的容器镜像环境；

默认：从中央容器仓拉取对应的容器镜像，运行实例并登陆。

### `ge config`

配置外部依赖服务器，具体参数如下：

```sh
$ ge config -h

Usage: ge config [OPTIONS]

update server config for ge, you need input all config info (ip, user, password)

Options:
-i, --ip Config ip config
-u, --user Config user name
-p, --password Config password
-h, --help

Example: ge config -i=121.36.**.** -u=asc**, -p=Asc***\#@\!$ (Need add escape character \ before special charactor $、#、!)
```

参数详细解释：

- `-i, --ip` : 配置依赖库服务器IP地址；
- `-u, --usr` : 配置依赖库服务器用户名；
- `-p, --password` : 配置依赖库地址；

默认：打印帮助信息。


### `ge update`

安装graph engine构建所需的外部依赖库，具体参数如下：

```sh
$ ge update -h

Usage: ge update [OPTIONS]

update dependencies of build and test

Options:
-d, --download Download dependencies
-i, --install Install dependencies
-c, --clear Clear dependencies
-h, --help
```

参数详细解释：

- `-d, --download` : 下载构建需要外部依赖库；
- `-i, --install` : 安装外部依赖包到对应位置；
- `-c, --clear` : 清除下载的外部依赖包；

默认：根据"scripts/update/deps_config.sh"的配置下载外部依赖库并安装到对应目录。
（注：请确保“scripts/update/server_config.sh”中的服务器地址、用户名、密码已经配置）

### `ge format`

使用clang-format进行代码格式化，具体参数如下：

```sh
$ ge format -h
Options:
-a format of all files
-c format of the files changed compared to last commit, default case
-l format of the files changed in last commit
-h Print usage
```

参数详细解释：

- `-a` : 格式化所有代码；
- `-c` : 只格式化本次修改的代码；
- `-l` : 格式化上次提交的代码；

默认：格式化本次修改代码。

### `ge lint`

使用clang-format进行代码格式化检查，具体参数如下：

```sh
$ ge lint -h

Options:
-a Check code format of all files, default case
-c Check code format of the files changed compared to last commit
-l Check code format of the files changed in last commit
-h Print usage
```

参数详细解释：

- `-a` : 检查所有代码格式；
- `-c` : 只检查修改的代码格式；
- `-l` : 检查上次提交的代码格式；

默认：检查本次修改代码格式。

### `ge build`

执行构建 (注：调用原有build.sh脚本，改造中...)；

### `ge test`

构建和运行测试用例，目前可以支持参数如下：

```sh
$ ge test -h

Usage: ge test [OPTIONS]

Options:
-u, --unit Run unit Test
-c, --component Run component Test
-h, --help
```

参数详细解释：

- `-u, --unit` : 执行单元测试
- `-c, --component` : 执行组件测试

默认：执行单元测试。

### `ge cov`

执行代码覆盖率检查, 支持全量覆盖和增量覆盖的功能，该命令需要已经跑完测试用例，目前可以支持参数如下：

```sh
$ ge cov -h

Usage: ge cov [OPTIONS]

Options:
-a, --all Full coverage
-i, --increment Increment coverage
-d, --directory Coverage of directory
-h, --help
```

参数详细解释：

- `-a, --all` : 执行全量覆盖率统计；
- `-i, --increment` : 执行增量覆盖率检查，默认是分析未提交修改的代码覆盖率（如果存在新增加的git未跟踪文件，需要先git add 添加进来才可以）；
- `-d, --directory` : 代码进行增量覆盖率检查的代码路径，支持传入路径参数；

默认：执行增量覆盖率检查；

下面的命令演示了如何检查ge目录下所有代码的增量覆盖率：

```sh
$ ge cov -d=ge
```

### `ge docs`

Doxygen文档生成，包含代码逻辑和物理结构和关系，方便阅读和理解代码；目前可以支持参数如下：

```sh
$ ge docs -h

Usage: ge docs [OPTIONS]

Options:
-b, --brief Build brief docs
-a, --all Build all docs
-h, --help
```

参数详细解释：

- `-b, --brief` : 生成简要文档，忽略部分关系图生成，速度快；
- `-a, --all` : 生成全量文档，包含各种代码关系图，速度相对慢；

默认： 生成全量代码文档。

### `ge clean`

清除各种下载或生成的中间文件，目前支持的参数如下：

```sh
$ ge clean -h

Usage: ge clean [OPTIONS]

Options:
-b, --build Clean build dir
-d, --docs Clean generate docs
-i, --install Clean dependenices
-a, --all Clean all
-h, --help
```

参数详细解释：

- `-b, --build` : 清除生成的编译构建临时文件；
- `-d, --docs` : 清除生成的文档临时文件；
- `-i, --install` : 清除安装的依赖文件；
- `-a, --all` : 清除所有下载和生成的临时文件；

默认：清除编译构建产生临时文件。

## Follow us

工具链的功能还在不断完善中，有问题请提issue，谢谢！

+ 80
- 0
scripts/test/ge_test.sh View File

@@ -0,0 +1,80 @@
#!/bin/bash
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
set -e

PROJECT_HOME=${PROJECT_HOME:-$(dirname "$0")/../../}

function help(){
cat <<-EOF
Usage: ge test [OPTIONS]

Options:
-u, --unit Run unit Test
-c, --component Run component Test
-h, --help
EOF

}

function unit_test(){
${PROJECT_HOME}/build.sh -u
}

function component_test(){
${PROJECT_HOME}/build.sh -s
}

function parse_args(){
parsed_args=$(getopt -a -o uch --long unit,component,help -- "$@") || {
help
exit 1
}

if [ $# -lt 1 ]; then
unit_test
exit 1
fi

eval set -- "$parsed_args"
while true; do
case "$1" in
-u | --unit)
unit_test
;;
-c | --component)
component_test
;;
-h | --help)
help
;;
--)
shift; break;
;;
*)
help; exit 1
;;
esac
shift
done
}

function main(){
parse_args "$@"
}

main "$@"

set +e

+ 47
- 0
scripts/update/deps_config.sh View File

@@ -0,0 +1,47 @@
#!/bin/bash
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
set -e

SERVER_CONFIG_FILE=${PROJECT_HOME}/scripts/config/server_config.sh

[ -e $SERVER_CONFIG_FILE ] || {
echo "You have not config dependencies account info first !!!!!"
${PROJECT_HOME}/scripts/config/ge_config.sh -h
exit 1;
}

source scripts/config/server_config.sh

CPU_ARCH=ubuntu18.04.x86_64
DRIVER_VERSION=20.2.0
CHIP_NAME=A800-9010
PRODUCT_VERSION=driver_C76_TR5

DRIVER_NAME=npu-driver
DRIVER_RUN_NAME=${CHIP_NAME}-${DRIVER_NAME}_${DRIVER_VERSION}_ubuntu18.04-x86_64.run

DEV_TOOLS_VERSION=1.78.t10.0.b100

export ATC_RUN_NAME=Ascend-atc-${DEV_TOOLS_VERSION}-${CPU_ARCH}.run
export ACL_RUN_NAME=Ascend-acllib-${DEV_TOOLS_VERSION}-${CPU_ARCH}.run
export FWKACL_RUN_NAME=Ascend-fwkacllib-${DEV_TOOLS_VERSION}-${CPU_ARCH}.run

DEV_TOOLS_PACKAGE=x86_ubuntu_os_devtoolset_package

export DRIVER_URL=${SERVER_PATH}/${PRODUCT_VERSION}/${DRIVER_RUN_NAME}
export DEV_TOOLS_URL=${SERVER_PATH}/20210428/${DEV_TOOLS_PACKAGE}.zip

set +e

+ 136
- 0
scripts/update/ge_update.sh View File

@@ -0,0 +1,136 @@
#!/bin/bash
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
set -e

PROJECT_HOME=${PROJECT_HOME:-$(dirname "$0")/../../}
PROJECT_HOME=$(cd $PROJECT_HOME || return; pwd)
DOWNLOAD_PATH=${PROJECT_HOME}/deps
DEP_LIB_DIR=./lib
DEP_TMP_DIR=./tmp

function extract_deps_so()
{
echo "begin to extract .run file ........."
chmod 777 ./${DRIVER_RUN_NAME}
unzip ${DEV_TOOLS_PACKAGE}.zip
chmod -R 777 ${DEV_TOOLS_PACKAGE}
[ -n "${DEP_TMP_DIR}" ] && rm -rf "${DEP_TMP_DIR}"
./${DRIVER_RUN_NAME} --noexec --extract=${DEP_TMP_DIR}/driver
./${DEV_TOOLS_PACKAGE}/${ATC_RUN_NAME} --noexec --extract=${DEP_TMP_DIR}/atc
./${DEV_TOOLS_PACKAGE}/${ACL_RUN_NAME} --noexec --extract=${DEP_TMP_DIR}/acllib
./${DEV_TOOLS_PACKAGE}/${FWKACL_RUN_NAME} --noexec --extract=${DEP_TMP_DIR}/fwkacllib
}

function copy_so_to_target_dir()
{
mkdir -p $DEP_LIB_DIR
mv ${DEP_TMP_DIR}/driver/driver $DEP_LIB_DIR/driver
mv ${DEP_TMP_DIR}/atc/atc $DEP_LIB_DIR/atc
mv ${DEP_TMP_DIR}/acllib/acllib $DEP_LIB_DIR/acllib
mv ${DEP_TMP_DIR}/fwkacllib/fwkacllib $DEP_LIB_DIR/fwkacllib
}

function clear_libs()
{
[ -n "${DOWNLOAD_PATH}" ] && rm -rf "${DOWNLOAD_PATH}"
}

function download_runs()
{
source scripts/update/deps_config.sh
echo "begin to download .run file ........."
clear_libs
mkdir -p ./ ${DOWNLOAD_PATH}
pushd "${DOWNLOAD_PATH}" >/dev/null
cd ${DOWNLOAD_PATH}
wget --user=${DEP_USER} --password=${DEP_PASSWORD} ${DRIVER_URL}
wget --user=${DEP_USER} --password=${DEP_PASSWORD} ${DEV_TOOLS_URL}
popd >/dev/null

}

function install_deps()
{
source scripts/update/deps_config.sh
mkdir -p ./ ${DOWNLOAD_PATH}
pushd "${DOWNLOAD_PATH}" >/dev/null
cd ${DOWNLOAD_PATH}
extract_deps_so
copy_so_to_target_dir
popd >/dev/null
}


function help(){
cat <<-EOF
Usage: ge update [OPTIONS]

update dependencies of build and test

Options:
-d, --download Download dependencies
-i, --install Install dependencies
-c, --clear Clear dependencies
-h, --help
EOF

}

function parse_args(){
parsed_args=$(getopt -a -o dich --long download,install,clear,help -- "$@") || {
help
exit 1
}

if [ $# -lt 1 ]; then
download_runs
install_deps
exit 1
fi

eval set -- "$parsed_args"
while true; do
case "$1" in
-d | --download)
download_runs
;;
-i | --install)
install_deps
;;
-c | --clear)
clear_libs
;;
-h | --help)
help; exit 1;
;;
--)
shift; break;
;;
*)
help; exit 1
;;
esac
shift
done
}

function main(){
parse_args "$@"
}

main "$@"

set +e

Write Preview
Loading…
Cancel
Save