From 9e14d6727b7583fed29f0684a1171754a505388d Mon Sep 17 00:00:00 2001 From: "mulin.lyh" Date: Fri, 2 Sep 2022 11:02:43 +0800 Subject: [PATCH] [to #44571845]fix: ci support multiple image Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/9974293 --- .dev_scripts/ci_container_test.sh | 3 - .dev_scripts/dockerci.sh | 5 +- requirements/tensorflow1x.txt | 1 + tests/isolated_cases.txt | 6 - tests/run.py | 191 ++++++++++++++++++++---------- tests/run_config.yaml | 31 +++++ 6 files changed, 165 insertions(+), 72 deletions(-) create mode 100644 requirements/tensorflow1x.txt delete mode 100644 tests/isolated_cases.txt create mode 100644 tests/run_config.yaml diff --git a/.dev_scripts/ci_container_test.sh b/.dev_scripts/ci_container_test.sh index 2f18aff7..a53c08c6 100644 --- a/.dev_scripts/ci_container_test.sh +++ b/.dev_scripts/ci_container_test.sh @@ -4,8 +4,6 @@ pip install -r requirements/cv.txt -f https://modelscope.oss-cn-beijing.aliyuncs pip install -r requirements/multi-modal.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html pip install -r requirements/nlp.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html pip install -r requirements/tests.txt -# install numpy<=1.18 for tensorflow==1.15.x -pip install "numpy<=1.18" git config --global --add safe.directory /Maas-lib @@ -26,4 +24,3 @@ else fi echo "Running case with command: $ci_command" $ci_command -#python tests/run.py --isolated_cases test_text_to_speech.py test_multi_modal_embedding.py test_ofa_tasks.py test_video_summarization.py diff --git a/.dev_scripts/dockerci.sh b/.dev_scripts/dockerci.sh index dbb79514..e76f2f14 100644 --- a/.dev_scripts/dockerci.sh +++ b/.dev_scripts/dockerci.sh @@ -7,7 +7,8 @@ gpus='7 6 5 4 3 2 1 0' cpu_sets='0-7 8-15 16-23 24-30 31-37 38-44 45-51 52-58' cpu_sets_arr=($cpu_sets) is_get_file_lock=false -CI_COMMAND=${CI_COMMAND:-bash .dev_scripts/ci_container_test.sh $RUN_CASE_COMMAND} +# export RUN_CASE_COMMAND='python tests/run.py --run_config tests/run_config.yaml' +CI_COMMAND=${CI_COMMAND:-bash .dev_scripts/ci_container_test.sh $RUN_CASE_BASE_COMMAND} echo "ci command: $CI_COMMAND" for gpu in $gpus do @@ -16,6 +17,7 @@ do echo "get gpu lock $gpu" CONTAINER_NAME="modelscope-ci-$gpu" let is_get_file_lock=true + # pull image if there are update docker pull ${IMAGE_NAME}:${IMAGE_VERSION} docker run --rm --name $CONTAINER_NAME --shm-size=16gb \ @@ -38,6 +40,7 @@ do --net host \ ${IMAGE_NAME}:${IMAGE_VERSION} \ $CI_COMMAND + if [ $? -ne 0 ]; then echo "Running test case failed, please check the log!" exit -1 diff --git a/requirements/tensorflow1x.txt b/requirements/tensorflow1x.txt new file mode 100644 index 00000000..b139efe1 --- /dev/null +++ b/requirements/tensorflow1x.txt @@ -0,0 +1 @@ +numpy==1.18.5 diff --git a/tests/isolated_cases.txt b/tests/isolated_cases.txt deleted file mode 100644 index be85142a..00000000 --- a/tests/isolated_cases.txt +++ /dev/null @@ -1,6 +0,0 @@ - test_text_to_speech.py - test_multi_modal_embedding.py - test_ofa_tasks.py - test_video_summarization.py - test_dialog_modeling.py - test_csanmt_translation.py diff --git a/tests/run.py b/tests/run.py index 79509745..478cb9d6 100644 --- a/tests/run.py +++ b/tests/run.py @@ -21,6 +21,7 @@ import pandas # if 'import tensorflow' in front of 'import torch'. # Puting a 'import torch' here can bypass this incompatibility. import torch +import yaml from modelscope.utils.logger import get_logger from modelscope.utils.test_utils import set_test_level, test_level @@ -61,6 +62,7 @@ def statistics_test_result(df): result, total_cases, success_cases, failures_cases, error_cases, skipped_cases, expected_failure_cases, unexpected_success_cases) + print('Testing result summary.') print(result_msg) if result == 'FAILED': sys.exit(1) @@ -88,6 +90,7 @@ def gather_test_suites_files(test_dir, pattern): for file in filenames: if fnmatch(file, pattern): case_file_list.append(file) + return case_file_list @@ -125,18 +128,6 @@ def collect_test_results(case_results): return result_list -class TestSuiteRunner: - - def run(self, msg_queue, test_dir, test_suite_file): - test_suite = unittest.TestSuite() - test_case = unittest.defaultTestLoader.discover( - start_dir=test_dir, pattern=test_suite_file) - test_suite.addTest(test_case) - runner = TimeCostTextTestRunner() - test_suite_result = runner.run(test_suite) - msg_queue.put(collect_test_results(test_suite_result)) - - def run_command_with_popen(cmd): with subprocess.Popen( cmd, @@ -148,55 +139,126 @@ def run_command_with_popen(cmd): sys.stdout.write(line) +def save_test_result(df, args): + if args.result_dir is not None: + file_name = str(int(datetime.datetime.now().timestamp() * 1000)) + os.umask(0) + Path(args.result_dir).mkdir(mode=0o777, parents=True, exist_ok=True) + Path(os.path.join(args.result_dir, file_name)).touch( + mode=0o666, exist_ok=True) + df.to_pickle(os.path.join(args.result_dir, file_name)) + + +def run_command(cmd): + logger.info('Running command: %s' % ' '.join(cmd)) + response = subprocess.run( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + try: + response.check_returncode() + logger.info(response.stdout.decode('utf8')) + except subprocess.CalledProcessError as error: + logger.error( + 'stdout: %s, stderr: %s' % + (response.stdout.decode('utf8'), error.stderr.decode('utf8'))) + + +def install_packages(pkgs): + cmd = [sys.executable, '-m', 'pip', 'install'] + for pkg in pkgs: + cmd.append(pkg) + + run_command(cmd) + + +def install_requirements(requirements): + for req in requirements: + cmd = [ + sys.executable, '-m', 'pip', 'install', '-r', + 'requirements/%s' % req, '-f', + 'https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html' + ] + run_command(cmd) + + +def run_case_in_env(env_name, env, test_suite_env_map, isolated_cases, + result_dir): + # install requirements and deps # run_config['envs'][env] + if 'requirements' in env: + install_requirements(env['requirements']) + if 'dependencies' in env: + install_packages(env['dependencies']) + + for test_suite_file in isolated_cases: # run case in subprocess + if test_suite_file in test_suite_env_map and test_suite_env_map[ + test_suite_file] == env_name: + cmd = [ + 'python', + 'tests/run.py', + '--pattern', + test_suite_file, + '--result_dir', + result_dir, + ] + run_command_with_popen(cmd) + else: + pass # case not in run list. + + # run remain cases in a process. + remain_suite_files = [] + for k, v in test_suite_env_map.items(): + if k not in isolated_cases and v == env_name: + remain_suite_files.append(k) + if len(remain_suite_files) == 0: + return + cmd = ['python', 'tests/run.py', '--result_dir', result_dir, '--suites'] + for suite in remain_suite_files: + cmd.append(suite) + run_command_with_popen(cmd) + + def run_in_subprocess(args): # only case args.isolated_cases run in subporcess, all other run in a subprocess test_suite_files = gather_test_suites_files( os.path.abspath(args.test_dir), args.pattern) + run_config = None + isolated_cases = [] + test_suite_env_map = {} + # put all the case in default env. + for test_suite_file in test_suite_files: + test_suite_env_map[test_suite_file] = 'default' + + if args.run_config is not None and Path(args.run_config).exists(): + with open(args.run_config) as f: + run_config = yaml.load(f, Loader=yaml.FullLoader) + if 'isolated' in run_config: + isolated_cases = run_config['isolated'] + + if 'envs' in run_config: + for env in run_config['envs']: + if env != 'default': + for test_suite in run_config['envs'][env]['tests']: + if test_suite in test_suite_env_map: + test_suite_env_map[test_suite] = env if args.subprocess: # run all case in subprocess isolated_cases = test_suite_files - else: - isolated_cases = [] - with open(args.isolated_cases, 'r') as f: - for line in f: - if line.strip() in test_suite_files: - isolated_cases.append(line.strip()) - - if not args.list_tests: - with tempfile.TemporaryDirectory() as temp_result_dir: - for test_suite_file in isolated_cases: # run case in subprocess - cmd = [ - 'python', 'tests/run.py', '--pattern', test_suite_file, - '--result_dir', temp_result_dir - ] - run_command_with_popen(cmd) - result_dfs = [] - # run remain cases in a process. - remain_suite_files = [ - item for item in test_suite_files if item not in isolated_cases - ] - test_suite = gather_test_suites_in_files(args.test_dir, - remain_suite_files, - args.list_tests) - if test_suite.countTestCases() > 0: - runner = TimeCostTextTestRunner() - result = runner.run(test_suite) - result = collect_test_results(result) - df = test_cases_result_to_df(result) - result_dfs.append(df) - # collect test results - result_path = Path(temp_result_dir) - for result in result_path.iterdir(): - if Path.is_file(result): - df = pandas.read_pickle(result) - result_dfs.append(df) + with tempfile.TemporaryDirectory() as temp_result_dir: + for env in set(test_suite_env_map.values()): + run_case_in_env(env, run_config['envs'][env], test_suite_env_map, + isolated_cases, temp_result_dir) - result_pd = pandas.concat( - result_dfs) # merge result of every test suite. - print_table_result(result_pd) - print_abnormal_case_info(result_pd) - statistics_test_result(result_pd) + result_dfs = [] + result_path = Path(temp_result_dir) + for result in result_path.iterdir(): + if Path.is_file(result): + df = pandas.read_pickle(result) + result_dfs.append(df) + result_pd = pandas.concat( + result_dfs) # merge result of every test suite. + print_table_result(result_pd) + print_abnormal_case_info(result_pd) + statistics_test_result(result_pd) def get_object_full_name(obj): @@ -293,15 +355,19 @@ def print_table_result(df): def main(args): runner = TimeCostTextTestRunner() - test_suite = gather_test_cases( - os.path.abspath(args.test_dir), args.pattern, args.list_tests) + if args.suites is not None and len(args.suites) > 0: + logger.info('Running: %s' % ' '.join(args.suites)) + test_suite = gather_test_suites_in_files(args.test_dir, args.suites, + args.list_tests) + else: + test_suite = gather_test_cases( + os.path.abspath(args.test_dir), args.pattern, args.list_tests) if not args.list_tests: result = runner.run(test_suite) result = collect_test_results(result) df = test_cases_result_to_df(result) if args.result_dir is not None: - file_name = str(int(datetime.datetime.now().timestamp() * 1000)) - df.to_pickle(os.path.join(args.result_dir, file_name)) + save_test_result(df, args) else: print_table_result(df) print_abnormal_case_info(df) @@ -321,9 +387,9 @@ if __name__ == '__main__': parser.add_argument( '--disable_profile', action='store_true', help='disable profiling') parser.add_argument( - '--isolated_cases', + '--run_config', default=None, - help='specified isolated cases config file') + help='specified case run config file(yaml file)') parser.add_argument( '--subprocess', action='store_true', @@ -332,6 +398,10 @@ if __name__ == '__main__': '--result_dir', default=None, help='Save result to directory, internal use only') + parser.add_argument( + '--suites', + nargs='*', + help='Run specified test suites(test suite file list)') args = parser.parse_args() set_test_level(args.level) os.environ['REGRESSION_BASELINE'] = '1' @@ -340,10 +410,7 @@ if __name__ == '__main__': from utils import profiler logger.info('enable profile ...') profiler.enable() - if args.isolated_cases is not None or args.subprocess: + if args.run_config is not None or args.subprocess: run_in_subprocess(args) - elif args.isolated_cases is not None and args.subprocess: - print('isolated_cases and subporcess conflict') - sys.exit(1) else: main(args) diff --git a/tests/run_config.yaml b/tests/run_config.yaml new file mode 100644 index 00000000..591dcd66 --- /dev/null +++ b/tests/run_config.yaml @@ -0,0 +1,31 @@ +# envs option allows fine-grained control for test executoin, for example, +# python tests/run.py --env pytorch +# would only trigger exeutions of all pytorch cases. +# envs option defaults to None for backward compatbility +isolated: # test cases that may require excessive anmount of GPU memory, which will be executed in dedicagted process. + - test_text_to_speech.py + - test_multi_modal_embedding.py + - test_ofa_tasks.py + - test_video_summarization.py + - test_dialog_modeling.py + - test_csanmt_translation.py + +envs: + default: # default env, case not in other env will in default, pytorch. + dependencies: # requirement packages，pip install before test case run. + - numpy>=1.20 + tensorflow1x: # cases excuted tensorflow1.x framework. + requirements: # requirements files run before test case run. + - tensorflow1x.txt + dependencies: # requirement packages，pip install before test case run. + - numpy==1.18.5 + tests: + - test_text_to_speech.py + - test_csanmt_translation.py + - test_translation_trainer.py + - test_ocr_detection.py + - test_automatic_speech_recognition.py + - test_image_matting.py + - test_person_image_cartoon.py + - test_skin_retouching.py + - test_image_style_transfer.py