!15990 fix cropper bug in Euler && add ci

From: @yeyunpeng2020
Reviewed-by: @zhang_xue_tong,@ddwsky
Signed-off-by: @zhang_xue_tong

mindspore/lite/test/model_config.cfg

@@ -0,0 +1,5 @@
+mobilenet_v1_0.25_128.tflite
+squeezenet.tflite
+hdc_Face_Emotion_MTI_Aesthetic.onnx
+porseg_tmp.onnx
+hdc_isface

mindspore/lite/test/run_benchmark_nets.sh

@@ -2663,6 +2663,18 @@ if [[ $backend == "all" || $backend == "arm_cpu" || $backend == "arm64_fp16" ]];
     sleep 1
 fi
 
+if [[ $backend == "all" || $backend == "gpu_npu" || $backend == "npu" ]]; then
+    cd ${basepath} || exit 1
+    bash ${basepath}/run_cropper.sh -r ${release_path} -d ${device_id} -m ${models_path}
+    Run_cropper_status=$?
+    if [[ ${Run_cropper_status} != 0 ]];then
+        echo "Run cropper failed"
+        cat ${run_npu_log_file}
+        isFailed=1
+        exit 1
+    fi
+fi
+
 if [[ $backend == "all" || $backend == "gpu_npu" || $backend == "gpu" ]]; then
     # Run on gpu
     arm64_path=${release_path}/android_aarch64

mindspore/lite/test/run_cropper.sh

@@ -0,0 +1,222 @@
+#!/bin/bash
+
+# Print start msg after run testcase
+function MS_PRINT_TESTCASE_END_MSG() {
+    echo -e "-----------------------------------------------------------------------------------------------------------------------------------"
+}
+# Print start msg before run testcase
+function MS_PRINT_TESTCASE_START_MSG() {
+    echo ""
+    echo -e "-----------------------------------------------------------------------------------------------------------------------------------"
+    echo -e "env                  Testcase                                                                                             Result   "
+    echo -e "---                  --------                                                                                             ------   "
+}
+
+function Print_Converter_Result() {
+    MS_PRINT_TESTCASE_END_MSG
+    while read line; do
+        arr=("${line}")
+        printf "%-15s %-20s %-90s %-7s\n" ${arr[0]} ${arr[1]} ${arr[2]} ${arr[3]}
+    done < "${run_converter_result_file}"
+    MS_PRINT_TESTCASE_END_MSG
+}
+
+function Print_Cropper_Result() {
+    MS_PRINT_TESTCASE_START_MSG
+    while read line; do
+        arr=("${line}")
+        printf "%-20s %-100s %-7s\n" ${arr[0]} ${arr[1]} ${arr[2]}
+    done < "${run_cropper_result}"
+    MS_PRINT_TESTCASE_END_MSG
+}
+
+# Run converter on x86 platform:
+function Run_Converter() {
+    # Unzip x86 runtime and converter
+    cd "${x86_path}" || exit 1
+    tar -zxf mindspore-lite-${version}-inference-linux-x64.tar.gz || exit 1
+    cd "${x86_path}"/mindspore-lite-${version}-inference-linux-x64/ || exit 1
+
+    cp tools/converter/converter/converter_lite ./ || exit 1
+    export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:./tools/converter/lib/:./tools/converter/third_party/glog/lib
+
+    rm -rf ${ms_models_path}
+    mkdir -p "${ms_models_path}"
+
+    # Convert models which does not need to be cared about the accuracy:
+    while read line; do
+        if [[ $line == \#* ]]; then
+          continue
+        fi
+        model_name=${line%%;*}
+        model_type=${model_name##*.}
+        case $model_type in
+          pb)
+            model_fmk="TF"
+            ;;
+          tflite)
+            model_fmk="TFLITE"
+            ;;
+          onnx)
+            model_fmk="ONNX"
+            ;;
+          mindir)
+            model_fmk="MINDIR"
+            ;;
+          *)
+            model_type="caffe"
+            model_fmk="CAFFE"
+            ;;
+        esac
+        if [[ $model_fmk == "CAFFE" ]]; then
+          echo ${model_name} >> "${run_converter_log_file}"
+          echo './converter_lite  --fmk='${model_fmk}' --modelFile='$models_path/${model_name}'.prototxt --weightFile='$models_path'/'${model_name}'.caffemodel --outputFile='${ms_models_path}'/'${model_name} >> "${run_converter_log_file}"
+          ./converter_lite  --fmk=${model_fmk} --modelFile=${models_path}/${model_name}.prototxt --weightFile=${models_path}/${model_name}.caffemodel --outputFile=${ms_models_path}/${model_name}
+        else
+          echo ${model_name} >> "${run_converter_log_file}"
+          echo './converter_lite  --fmk='${model_fmk}' --modelFile='${models_path}'/'${model_name}' --outputFile='${ms_models_path}'/'${model_name} >> "${run_converter_log_file}"
+          ./converter_lite  --fmk=${model_fmk} --modelFile=${models_path}/${model_name} --outputFile=${ms_models_path}/${model_name}
+        fi
+        if [ $? = 0 ]; then
+            converter_result='converter '${model_type}' '${model_name}' pass';echo ${converter_result} >> "${run_converter_result_file}"
+        else
+            converter_result='converter '${model_type}' '${model_name}' failed';echo ${converter_result} >> "${run_converter_result_file}";return 1
+        fi
+    done < ${cropper_config}
+}
+
+function Run_cropper() {
+    cd ${arm64_path} || exit 1
+    tar -zxf mindspore-lite-${version}-inference-android-aarch64.tar.gz || exit 1
+    cd mindspore-lite-${version}-inference-android-aarch64 || exit 1
+    cp -a ./inference/third_party/hiai_ddk/lib/libhiai.so "${cropper_test_path}"/libhiai.so || exit 1
+    cp -a ./inference/third_party/hiai_ddk/lib/libhiai_ir.so "${cropper_test_path}"/libhiai_ir.so || exit 1
+    cp -a ./inference/third_party/hiai_ddk/lib/libhiai_ir_build.so "${cropper_test_path}"/libhiai_ir_build.so || exit 1
+
+    cp -a ./inference/lib/libmindspore-lite.a "${cropper_test_path}"/libmindspore-lite.a || exit 1
+    cp -a ./tools/benchmark/benchmark "${cropper_test_path}"/benchmark || exit 1
+
+    cp -r "${x86_path}"/mindspore-lite-${version}-inference-linux-x64/tools/cropper/ "${cropper_test_path}" || exit 1
+
+    cd "${cropper_test_path}" || exit 1
+    echo "${cropper_test_path}"
+
+    # adb push all needed files to the phone
+    adb -s ${device_id} push "${cropper_test_path}" /data/local/tmp/ > adb_push_log.txt
+
+    # run adb ,run session ,check the result:
+    echo 'rm -rf /data/local/tmp/cropper_test' > adb_cmd.txt
+    echo 'cd  /data/local/tmp/cropper_test' > adb_cmd.txt
+    echo 'cp  /data/local/tmp/libc++_shared.so ./' >> adb_cmd.txt
+    echo 'chmod 777 benchmark' >> adb_cmd.txt
+
+    adb -s ${device_id} shell < adb_cmd.txt
+
+    while read line; do
+        model_line_info=${line}
+        if [[ $model_line_info == \#* ]]; then
+          continue
+        fi
+        model_name=`echo ${line}|awk -F ' ' '{print $1}'`
+
+        echo "./cropper/cropper --packageFile=./libmindspore-lite.a --configFile=./cropper/cropper_mapping_npu.cfg --modelFile=${ms_models_path}/${model_name}.ms --outputFile=./libmindspore-lite-${model_name}.a"
+         ./cropper/cropper --packageFile=./libmindspore-lite.a --configFile=./cropper/cropper_mapping_npu.cfg --modelFile=${ms_models_path}/${model_name}.ms --outputFile=./libmindspore-lite-${model_name}.a
+
+        if [ $? = 0 ]; then
+            run_result='cropper_lib: '${line}' pass'; echo ${run_result} >> "${run_cropper_result}"
+        else
+            run_result='cropper_lib: '${line}' failed'; echo ${run_result} >> "${run_cropper_result}"; return 1
+        fi
+        
+       "${ANDROID_NDK}"/toolchains/llvm/prebuilt/linux-x86_64/bin/clang++ -Wl,--whole-archive ./libmindspore-lite-${model_name}.a \
+       --target=aarch64-none-linux-android21 \
+       --gcc-toolchain="${ANDROID_NDK}"/toolchains/llvm/prebuilt/linux-x86_64 \
+       --sysroot="${ANDROID_NDK}"/toolchains/llvm/prebuilt/linux-x86_64/sysroot -fPIC -fPIC -fPIE -D_FORTIFY_SOURCE=2 \
+       -O2 -Wall  -fstack-protector-all -Wno-attributes -Wno-deprecated-declarations -Wno-missing-braces -Wno-overloaded-virtual \
+       -g -DANDROID -fdata-sections -ffunction-sections -funwind-tables -fstack-protector-strong -no-canonical-prefixes \
+       -fno-addrsig -Wa,--noexecstack -Wformat -Werror=format-security   -O0 -std=c++17 -DARM64=1 -O2 -DNDEBUG  -s \
+       -Wl,--exclude-libs,libgcc.a -Wl,--exclude-libs,libatomic.a -Wl,--build-id -Wl,--warn-shared-textrel \
+       -Wl,--fatal-warnings -Wl,--no-undefined -Qunused-arguments -Wl,-z,noexecstack -Wl,--no-whole-archive -llog -latomic -lm \
+       -L "${cropper_test_path}" -lhiai -lhiai_ir -lhiai_ir_build \
+       -shared -o libmindspore-lite.so -Wl,-soname,libmindspore-lite.so
+        
+        if [ $? = 0 ]; then
+            run_result='link_lib_to_so: '${line}' pass'; echo ${run_result} >> "${run_cropper_result}"
+        else
+            run_result='link_lib_to_so: '${line}' failed'; echo ${run_result} >> "${run_cropper_result}"; return 1
+        fi
+        adb -s ${device_id} push "${cropper_test_path}"/libmindspore-lite.so /data/local/tmp/cropper_test > adb_push_log.txt
+
+        echo "mindspore run cropper: ${model_name}, accuracy limit:4" >> "${run_cropper_log_file}"
+        echo 'cd  /data/local/tmp/cropper_test' > adb_run_cmd.txt
+        echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/cropper_test;./benchmark --device=NPU --modelFile=/data/local/tmp/benchmark_test/'${model_name}'.ms --loopCount=1 --warmUpLoopCount=0' >> "${run_cropper_log_file}"
+        echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/data/local/tmp/cropper_test;./benchmark --device=NPU --modelFile=/data/local/tmp/benchmark_test/'${model_name}'.ms --loopCount=1 --warmUpLoopCount=0' >> adb_run_cmd.txt
+        adb -s ${device_id} shell < adb_run_cmd.txt >> "${run_cropper_log_file}"
+        if [ $? = 0 ]; then
+            run_result='run_benchmark: '${model_name}' pass'; echo ${run_result} >> "${run_cropper_result}"
+        else
+            run_result='run_benchmark: '${model_name}' failed'; echo ${run_result} >> "${run_cropper_result}"; return 1
+        fi
+    done < ${cropper_config}
+}
+
+basepath=$(pwd)
+echo "${basepath}"
+
+# Example:sh run_cropper_nets.sh -r /home/temp_test -d "8KE5T19620002408"
+while getopts "r:d:m:" opt; do
+    case ${opt} in
+        r)
+            release_path=${OPTARG}
+            echo "release_path is ${OPTARG}"
+            ;;
+        d)
+            device_id=${OPTARG}
+            echo "device_id is ${OPTARG}"
+            ;;
+        m)
+            models_path=${OPTARG}
+            echo "models_path is ${OPTARG}"
+            ;;
+        ?)
+        echo "unknown para"
+        exit 1;;
+    esac
+done
+
+cropper_test_path="${basepath}"/cropper_test
+rm -rf "${cropper_test_path}"
+mkdir -p "${cropper_test_path}"
+
+run_cropper_result="${basepath}"/run_cropper_result.txt
+echo ' ' > "${run_cropper_result}"
+run_cropper_log_file="${basepath}"/run_cropper_log.txt
+echo 'run cropper logs: ' > "${run_cropper_log_file}"
+
+cropper_config="${basepath}"/model_config.cfg
+arm64_path=${release_path}/android_aarch64
+x86_path=${release_path}/ubuntu_x86
+
+# Write converter result to temp file
+run_converter_log_file="${basepath}"/run_converter_log.txt
+echo ' ' > "${run_converter_log_file}"
+
+run_converter_result_file="${basepath}"/run_converter_result.txt
+echo ' ' > "${run_converter_result_file}"
+
+file_name=$(ls "${x86_path}"/*inference-linux-x64.tar.gz)
+IFS="-" read -r -a file_name_array <<< "$file_name"
+version=${file_name_array[2]}
+ms_models_path=${basepath}/ms_models
+
+Run_cropper
+Run_cropper_status=$?
+
+if [[ $Run_cropper_status == 1 ]]; then
+    cat "${run_cropper_log_file}"
+    Print_Cropper_Result
+    exit 1
+fi
+
+Print_Cropper_Result
+exit 0

mindspore/lite/tools/cropper/build_cropper_config.sh

@@ -49,7 +49,7 @@ getDeep() {
   map_files=$(gcc -MM ${2} ${DEFINE_STR} ${HEADER_LOCATION})
   # first is *.o second is *.cc
   array_deep=()
-  while IFS='' read -r line; do array_deep+=("$line"); done < <(echo ${map_files} | awk -F '\' '{for(i=3;i<=NF;i++){print $i}}' | grep -E 'src/runtime|nnacl' | egrep -v ${REMOVE_LISTS_STR})
+  while IFS='' read -r line; do array_deep+=("$line"); done < <(echo ${map_files} | awk -F '\' '{for(i=3;i<=NF;i++){print $i}}' | egrep -v 'flatbuffers|build' | egrep -v ${REMOVE_LISTS_STR})
   # shellcheck disable=SC2068
   for array_deep_file in ${array_deep[@]}; do
     # only add existing files
@@ -103,32 +103,33 @@ getOpsFile() {
 
 getCommonFile() {
   echo "start get common files"
+  cd "${MINDSPORE_HOME}" || exit 1
   include_h=()
-  while IFS='' read -r line; do include_h+=("$line"); done < <(ls ${MINDSPORE_HOME}/mindspore/lite/include/*.h)
+  while IFS='' read -r line; do include_h+=("$line"); done < <(ls mindspore/lite/include/*.h)
   src_files_h=()
-  while IFS='' read -r line; do src_files_h+=("$line"); done < <(ls ${MINDSPORE_HOME}/mindspore/lite/src/*.h)
+  while IFS='' read -r line; do src_files_h+=("$line"); done < <(ls mindspore/lite/src/*.h)
   common_files_h=()
-  while IFS='' read -r line; do common_files_h+=("$line"); done < <(ls ${MINDSPORE_HOME}/mindspore/lite/src/common/*.h)
+  while IFS='' read -r line; do common_files_h+=("$line"); done < <(ls mindspore/lite/src/common/*.h)
   runtime_files_h=()
-  while IFS='' read -r line; do runtime_files_h+=("$line"); done < <(ls ${MINDSPORE_HOME}/mindspore/lite/src/runtime/*.h)
+  while IFS='' read -r line; do runtime_files_h+=("$line"); done < <(ls mindspore/lite/src/runtime/*.h)
   others_files_h=(
-    "${MINDSPORE_HOME}"/mindspore/lite/src/populate/populate_register.h
-    "${MINDSPORE_HOME}"/mindspore/lite/src/runtime/infer_manager.h
-    "${MINDSPORE_HOME}"/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/infer/infer_register.h
-    "${MINDSPORE_HOME}"/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/nnacl_utils.h
-    "${MINDSPORE_HOME}"/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/pack.h
-    "${MINDSPORE_HOME}"/mindspore/lite/src/runtime/kernel/arm/fp16/common_fp16.h
-    "${MINDSPORE_HOME}"/mindspore/lite/src/ops/populate/populate_register.h
-    "${MINDSPORE_HOME}"/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/op_base.h
-    "${MINDSPORE_HOME}"/mindspore/core/ir/dtype/type_id.h
-    "${MINDSPORE_HOME}"/mindspore/core/utils/overload.h
-    "${MINDSPORE_HOME}"/mindspore/lite/tools/common/option.h
-    "${MINDSPORE_HOME}"/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/intrinsics/ms_simd_instructions.h
-    "${MINDSPORE_HOME}"/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/intrinsics/ms_simd_instructions_fp16.h
-    "${MINDSPORE_HOME}"/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/infer/infer.h
-    "${MINDSPORE_HOME}"/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/tensor_c.h
-    "${MINDSPORE_HOME}"/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/infer/common_infer.h
-    "${MINDSPORE_HOME}"/mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/errorcode.h
+    mindspore/lite/src/populate/populate_register.h
+    mindspore/lite/src/runtime/infer_manager.h
+    mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/infer/infer_register.h
+    mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/nnacl_utils.h
+    mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/pack.h
+    mindspore/lite/src/runtime/kernel/arm/fp16/common_fp16.h
+    mindspore/lite/src/ops/populate/populate_register.h
+    mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/op_base.h
+    mindspore/core/ir/dtype/type_id.h
+    mindspore/core/utils/overload.h
+    mindspore/lite/tools/common/option.h
+    mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/intrinsics/ms_simd_instructions.h
+    mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/intrinsics/ms_simd_instructions_fp16.h
+    mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/infer/infer.h
+    mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/tensor_c.h
+    mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/infer/common_infer.h
+    mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/errorcode.h
   )
   all_files_h=("${include_h[@]}" "${src_files_h[@]}" "${common_files_h[@]}" "${runtime_files_h[@]}" "${others_files_h[@]}")