diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..9c607acc --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +*.png filter=lfs diff=lfs merge=lfs -text +*.jpg filter=lfs diff=lfs merge=lfs -text +*.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore index 3e6a3f4a..c8a1c717 100644 --- a/.gitignore +++ b/.gitignore @@ -104,7 +104,6 @@ venv.bak/ # mypy .mypy_cache/ -data .vscode .idea diff --git a/data/test/images/image1.jpg b/data/test/images/image1.jpg new file mode 100644 index 00000000..450a969d --- /dev/null +++ b/data/test/images/image1.jpg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78094cc48fbcfd9b6d321fe13619ecc72b65e006fc1b4c4458409ade9979486d +size 129862 diff --git a/data/test/images/image_matting.png b/data/test/images/image_matting.png new file mode 100644 index 00000000..de3f1918 --- /dev/null +++ b/data/test/images/image_matting.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af83a94899a6d23339c3ecc5c4c58c57c835af57b531a2f4c50461184f820141 +size 603621 diff --git a/docs/source/develop.md b/docs/source/develop.md index f96590b0..96120088 100644 --- a/docs/source/develop.md +++ b/docs/source/develop.md @@ -91,6 +91,55 @@ make tests 4. Daily regression tests will run all cases at 0 am each day using master branch. +### 2.3 Test data storage + +As we need a lot of data for testing, including images, videos, models. We use git lfs +to store those large files. + +1. install git-lfs +for mac +```bash +brew install git-lfs +git lfs install +``` + +for centos, please download rpm from git-lfs github release [website](https://github.com/git-lfs/git-lfs/releases/tag/v3.2.0) +```bash +wget http://101374-public.oss-cn-hangzhou-zmf.aliyuncs.com/git-lfs-3.2.0-1.el7.x86_64.rpm +sudo rpm -ivh git-lfs-3.2.0-1.el7.x86_64.rpm +git lfs install +``` + +for ubuntu +```bash +curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash +sudo apt-get install git-lfs +git lfs install +``` + +2. track your data type using git lfs, for example, to track png files +```bash +git lfs track "*.png" +``` + +3. add your test files to `data/test/` folder, you can make directories if you need. +```bash +git add data/test/test.png +``` + +4. commit your test data to remote branch +```bash +git commit -m "xxx" +``` + +To pull data from remote repo, just as the same way you pull git files. +```bash +git pull origin branch_name +``` + + + + ## Code Review 1. Run following command to create an aone CR, replace `TARGET_BRANCH` and `CR_NAME` with the one you want. diff --git a/tests/pipelines/test_base.py b/tests/pipelines/test_base.py index 73aebfdf..c642ed4b 100644 --- a/tests/pipelines/test_base.py +++ b/tests/pipelines/test_base.py @@ -80,8 +80,7 @@ class CustomPipelineTest(unittest.TestCase): pipe2 = pipeline(dummy_task) self.assertTrue(type(pipe) is type(pipe2)) - img_url = 'http://pai-vision-data-hz.oss-cn-zhangjiakou.' \ - 'aliyuncs.com/data/test/images/image1.jpg' + img_url = 'data/test/images/image1.jpg' output = pipe(img_url) self.assertEqual(output['filename'], img_url) self.assertEqual(output['output_png'].shape, (318, 512, 3)) diff --git a/tests/pipelines/test_image_captioning.py b/tests/pipelines/test_image_captioning.py index 4fac4658..74a65806 100644 --- a/tests/pipelines/test_image_captioning.py +++ b/tests/pipelines/test_image_captioning.py @@ -27,9 +27,7 @@ class ImageCaptionTest(unittest.TestCase): img_captioning = pipeline( Tasks.image_captioning, model=ofile.name, bpe_dir=bpe_dir) - result = img_captioning( - 'http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/data/test/maas/image_matting/test.png' - ) + result = img_captioning('data/test/images/image_matting.png') print(result['caption']) diff --git a/tests/pipelines/test_image_matting.py b/tests/pipelines/test_image_matting.py index 676153bf..6e102d00 100644 --- a/tests/pipelines/test_image_matting.py +++ b/tests/pipelines/test_image_matting.py @@ -34,16 +34,12 @@ class ImageMattingTest(unittest.TestCase): ofile.write(File.read(model_path)) img_matting = pipeline(Tasks.image_matting, model=tmp_dir) - result = img_matting( - 'http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/data/test/maas/image_matting/test.png' - ) + result = img_matting('data/test/images/image_matting.png') cv2.imwrite('result.png', result['output_png']) @unittest.skipUnless(test_level() >= 1, 'skip test in current test level') def test_run_with_dataset(self): - input_location = [ - 'http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/data/test/maas/image_matting/test.png' - ] + input_location = ['data/test/images/image_matting.png'] # alternatively: # input_location = '/dir/to/images' @@ -58,9 +54,7 @@ class ImageMattingTest(unittest.TestCase): def test_run_modelhub(self): img_matting = pipeline(Tasks.image_matting, model=self.model_id) - result = img_matting( - 'http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/data/test/maas/image_matting/test.png' - ) + result = img_matting('data/test/images/image_matting.png') cv2.imwrite('result.png', result['output_png']) print(f'Output written to {osp.abspath("result.png")}') @@ -68,9 +62,7 @@ class ImageMattingTest(unittest.TestCase): def test_run_modelhub_default_model(self): img_matting = pipeline(Tasks.image_matting) - result = img_matting( - 'http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/data/test/maas/image_matting/test.png' - ) + result = img_matting('data/test/images/image_matting.png') cv2.imwrite('result.png', result['output_png']) print(f'Output written to {osp.abspath("result.png")}') diff --git a/tests/preprocessors/test_image.py b/tests/preprocessors/test_image.py index cfa7b11d..21ae780e 100644 --- a/tests/preprocessors/test_image.py +++ b/tests/preprocessors/test_image.py @@ -11,9 +11,7 @@ from modelscope.utils.logger import get_logger class ImagePreprocessorTest(unittest.TestCase): def test_load(self): - img = load_image( - 'http://pai-vision-data-hz.oss-cn-zhangjiakou.aliyuncs.com/data/test/maas/image_matting/test.png' - ) + img = load_image('data/test/images/image_matting.png') self.assertTrue(isinstance(img, PIL.Image.Image)) self.assertEqual(img.size, (948, 533))