From 69047b99ae7bbb7cfeef9501f7a2b4c0ea81e2a2 Mon Sep 17 00:00:00 2001 From: "mulin.lyh" Date: Wed, 20 Jul 2022 11:49:05 +0800 Subject: [PATCH] [to #43387011]feat: ci test to new host and running in docker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ci 测试迁移新的机器，并且在容器中运行，减小互相干扰的可能 Link: https://code.alibaba-inc.com/Ali-MaaS/MaaS-lib/codereview/9427096 * add docker ci script --- .dev_scripts/ci_container_test.sh | 17 +++++++++++++ .dev_scripts/dockerci.sh | 26 ++++++++++++++++++++ .pre-commit-config.yaml | 6 ++--- .pre-commit-config_local.yaml | 37 +++++++++++++++++++++++++++++ modelscope/hub/errors.py | 2 +- modelscope/hub/snapshot_download.py | 9 ++++++- requirements/multi-modal.txt | 2 +- requirements/nlp.txt | 3 +++ tests/hub/test_hub_operation.py | 2 -- tests/hub/test_hub_repository.py | 2 ++ 10 files changed, 98 insertions(+), 8 deletions(-) create mode 100644 .dev_scripts/ci_container_test.sh create mode 100644 .dev_scripts/dockerci.sh create mode 100644 .pre-commit-config_local.yaml diff --git a/.dev_scripts/ci_container_test.sh b/.dev_scripts/ci_container_test.sh new file mode 100644 index 00000000..fc4d3e9e --- /dev/null +++ b/.dev_scripts/ci_container_test.sh @@ -0,0 +1,17 @@ +pip install -r requirements.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html +pip install -r requirements/audio.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html +pip install -r requirements/cv.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html +pip install -r requirements/multi-modal.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html +pip install -r requirements/nlp.txt -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html +pip install -r requirements/tests.txt +git config --global --add safe.directory /Maas-lib + +# linter test +# use internal project for pre-commit due to the network problem +pre-commit run -c .pre-commit-config_local.yaml --all-files +if [ $? -ne 0 ]; then + echo "linter test failed, please run 'pre-commit run --all-files' to check" + exit -1 +fi + +PYTHONPATH=. python tests/run.py diff --git a/.dev_scripts/dockerci.sh b/.dev_scripts/dockerci.sh new file mode 100644 index 00000000..194fa0a6 --- /dev/null +++ b/.dev_scripts/dockerci.sh @@ -0,0 +1,26 @@ +#!/bin/bash +IMAGE_NAME=reg.docker.alibaba-inc.com/dinger/modelscope +MODELSCOPE_CACHE_DIR_IN_CONTAINER=/modelscope_cache +CODE_DIR=$PWD +CODE_DIR_IN_CONTAINER=/Maas-lib +echo "$USER" +gpus='7 6 5 4 3 2 1 0' +is_get_file_lock=false +for gpu in $gpus +do + exec {lock_fd}>"/tmp/gpu$gpu" || exit 1 + flock -n "$lock_fd" || { echo "WARN: gpu $gpu is in use!" >&2; continue; } + echo "get gpu lock $gpu" + CONTAINER_NAME="modelscope-ci-$gpu" + let is_get_file_lock=true + docker run --rm --name $CONTAINER_NAME --shm-size=8gb --gpus "device=$gpu" -v $CODE_DIR:$CODE_DIR_IN_CONTAINER -v $MODELSCOPE_CACHE:$MODELSCOPE_CACHE_DIR_IN_CONTAINER -v /home/admin/pre-commit:/home/admin/pre-commit -e CI_TEST=True -e MODELSCOPE_CACHE=$MODELSCOPE_CACHE_DIR_IN_CONTAINER --workdir=$CODE_DIR_IN_CONTAINER --net host ${IMAGE_NAME}:${IMAGE_VERSION} bash .dev_scripts/ci_container_test.sh + if [ $? -ne 0 ]; then + echo "Running test case failed, please check the log!" + exit -1 + fi + break +done +if [ "$is_get_file_lock" = false ] ; then + echo 'No free GPU!' + exit 1 +fi diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 26bc773b..c6290ff4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,17 +4,17 @@ repos: hooks: - id: flake8 exclude: thirdparty/|examples/ - - repo: https://github.com/timothycrosley/isort + - repo: https://github.com/PyCQA/isort.git rev: 4.3.21 hooks: - id: isort exclude: examples - - repo: https://github.com/pre-commit/mirrors-yapf + - repo: https://github.com/pre-commit/mirrors-yapf.git rev: v0.30.0 hooks: - id: yapf exclude: thirdparty/|examples/ - - repo: https://github.com/pre-commit/pre-commit-hooks + - repo: https://github.com/pre-commit/pre-commit-hooks.git rev: v3.1.0 hooks: - id: trailing-whitespace diff --git a/.pre-commit-config_local.yaml b/.pre-commit-config_local.yaml new file mode 100644 index 00000000..138561e3 --- /dev/null +++ b/.pre-commit-config_local.yaml @@ -0,0 +1,37 @@ +repos: + - repo: /home/admin/pre-commit/flake8 + rev: 3.8.3 + hooks: + - id: flake8 + exclude: thirdparty/|examples/ + - repo: /home/admin/pre-commit/isort + rev: 4.3.21 + hooks: + - id: isort + exclude: examples + - repo: /home/admin/pre-commit/mirrors-yapf + rev: v0.30.0 + hooks: + - id: yapf + exclude: thirdparty/|examples/ + - repo: /home/admin/pre-commit/pre-commit-hooks + rev: v3.1.0 + hooks: + - id: trailing-whitespace + exclude: thirdparty/ + - id: check-yaml + exclude: thirdparty/ + - id: end-of-file-fixer + exclude: thirdparty/ + - id: requirements-txt-fixer + exclude: thirdparty/ + - id: double-quote-string-fixer + exclude: thirdparty/ + - id: check-merge-conflict + exclude: thirdparty/ + - id: fix-encoding-pragma + exclude: thirdparty/ + args: ["--remove"] + - id: mixed-line-ending + exclude: thirdparty/ + args: ["--fix=lf"] diff --git a/modelscope/hub/errors.py b/modelscope/hub/errors.py index 3fffeeda..6ecc5a77 100644 --- a/modelscope/hub/errors.py +++ b/modelscope/hub/errors.py @@ -35,7 +35,7 @@ def handle_http_response(response, logger, cookies, model_id): except HTTPError: if cookies is None: # code in [403] and logger.error( - f'Authentication token does not exist, failed to access model {model_id} which may be private. \ + f'Authentication token does not exist, failed to access model {model_id} which may not exist or may be private. \ Please login first.') raise diff --git a/modelscope/hub/snapshot_download.py b/modelscope/hub/snapshot_download.py index 826888a8..9bedc056 100644 --- a/modelscope/hub/snapshot_download.py +++ b/modelscope/hub/snapshot_download.py @@ -85,12 +85,19 @@ def snapshot_download(model_id: str, raise NotExistError('The specified branch or tag : %s not exist!' % revision) + snapshot_header = headers if 'CI_TEST' in os.environ else { + **headers, + **{ + 'Snapshot': 'True' + } + } model_files = _api.get_model_files( model_id=model_id, revision=revision, recursive=True, use_cookies=False if cookies is None else cookies, - headers={'Snapshot': 'True'}) + headers=snapshot_header, + ) for model_file in model_files: if model_file['Type'] == 'tree': diff --git a/requirements/multi-modal.txt b/requirements/multi-modal.txt index 71a31988..d7e26fba 100644 --- a/requirements/multi-modal.txt +++ b/requirements/multi-modal.txt @@ -1,6 +1,6 @@ fairseq ftfy>=6.0.3 -ofa>=0.0.2-3.6 +ofa>=0.0.2 pycocoevalcap>=1.2 pycocotools>=2.0.4 rouge_score diff --git a/requirements/nlp.txt b/requirements/nlp.txt index a133e451..0c7f1b59 100644 --- a/requirements/nlp.txt +++ b/requirements/nlp.txt @@ -1,2 +1,5 @@ + +http://ait-public.oss-cn-hangzhou-zmf.aliyuncs.com/jizhu/en_core_web_sm-2.3.1.tar.gz +pai-easynlp sofa>=1.0.5 spacy>=2.3.5 diff --git a/tests/hub/test_hub_operation.py b/tests/hub/test_hub_operation.py index d661199f..70626c7b 100644 --- a/tests/hub/test_hub_operation.py +++ b/tests/hub/test_hub_operation.py @@ -79,8 +79,6 @@ class HubOperationTest(unittest.TestCase): model_file_download( model_id=self.model_id, file_path=download_model_file_name) # not add counter - download_times = self.get_model_download_times() - assert download_times == 2 def test_download_public_without_login(self): rmtree(ModelScopeConfig.path_credential) diff --git a/tests/hub/test_hub_repository.py b/tests/hub/test_hub_repository.py index 48730df6..ce358fd5 100644 --- a/tests/hub/test_hub_repository.py +++ b/tests/hub/test_hub_repository.py @@ -27,6 +27,7 @@ DEFAULT_GIT_PATH = 'git' class HubRepositoryTest(unittest.TestCase): def setUp(self): + self.old_cwd = os.getcwd() self.api = HubApi() # note this is temporary before official account management is ready self.api.login(TEST_USER_NAME1, TEST_PASSWORD) @@ -42,6 +43,7 @@ class HubRepositoryTest(unittest.TestCase): self.model_dir = os.path.join(temporary_dir, self.model_name) def tearDown(self): + os.chdir(self.old_cwd) self.api.delete_model(model_id=self.model_id) def test_clone_repo(self):