From 927a3e746db81d2451bec1b0b24f7d936e388a34 Mon Sep 17 00:00:00 2001 From: Yige Xu Date: Tue, 17 Sep 2019 20:11:22 +0800 Subject: [PATCH] update test codes for testing classification and matching loader and pipe --- fastNLP/io/pipe/matching.py | 5 ++++- test/io/loader/test_classification_loader.py | 13 ++++++------- test/io/loader/test_matching_loader.py | 1 + test/io/pipe/test_classification.py | 4 +++- test/io/pipe/test_matching.py | 6 +++--- 5 files changed, 17 insertions(+), 12 deletions(-) diff --git a/fastNLP/io/pipe/matching.py b/fastNLP/io/pipe/matching.py index 6bf81085..aa6db46f 100644 --- a/fastNLP/io/pipe/matching.py +++ b/fastNLP/io/pipe/matching.py @@ -7,6 +7,9 @@ __all__ = [ "QuoraBertPipe", "QNLIBertPipe", "MNLIBertPipe", + "XNLIBertPipe", + "BQCorpusBertPipe", + "LCQMCBertPipe", "MatchingPipe", "RTEPipe", "SNLIPipe", @@ -15,7 +18,7 @@ __all__ = [ "MNLIPipe", "XNLIPipe", "BQCorpusPipe", - "LCQMCPipe" + "LCQMCPipe", ] import warnings diff --git a/test/io/loader/test_classification_loader.py b/test/io/loader/test_classification_loader.py index d866edec..f4ecd47d 100644 --- a/test/io/loader/test_classification_loader.py +++ b/test/io/loader/test_classification_loader.py @@ -1,15 +1,12 @@ import unittest -from fastNLP.io import DataBundle -from fastNLP.io.loader.classification import YelpFullLoader -from fastNLP.io.loader.classification import YelpPolarityLoader -from fastNLP.io.loader.classification import IMDBLoader -from fastNLP.io.loader.classification import SST2Loader -from fastNLP.io.loader.classification import SSTLoader -from fastNLP.io.loader.classification import ChnSentiCorpLoader import os +from fastNLP.io import DataBundle +from fastNLP.io.loader.classification import YelpFullLoader, YelpPolarityLoader, IMDBLoader, \ + SSTLoader, SST2Loader, ChnSentiCorpLoader, THUCNewsLoader, WeiboSenti100kLoader + @unittest.skipIf('TRAVIS' in os.environ, "Skip in travis") class TestDownload(unittest.TestCase): @@ -32,6 +29,8 @@ class TestLoad(unittest.TestCase): 'sst': ('test/data_for_tests/io/SST', SSTLoader, (6, 6, 6), False), 'imdb': ('test/data_for_tests/io/imdb', IMDBLoader, (6, 6, 6), False), 'ChnSentiCorp': ('test/data_for_tests/io/ChnSentiCorp', ChnSentiCorpLoader, (6, 6, 6), False), + 'THUCNews': ('test/data_for_tests/io/THUCNews', THUCNewsLoader, (9, 9, 9), False), + 'WeiboSenti100k': ('test/data_for_tests/io/WeiboSenti100k', WeiboSenti100kLoader, (7, 6, 6), False), } for k, v in data_set_dict.items(): path, loader, data_set, warns = v diff --git a/test/io/loader/test_matching_loader.py b/test/io/loader/test_matching_loader.py index eb4ec2ba..5700ab80 100644 --- a/test/io/loader/test_matching_loader.py +++ b/test/io/loader/test_matching_loader.py @@ -29,6 +29,7 @@ class TestMatchingLoad(unittest.TestCase): 'SNLI': ('test/data_for_tests/io/SNLI', SNLILoader, (5, 5, 5), False), 'QNLI': ('test/data_for_tests/io/QNLI', QNLILoader, (5, 5, 5), True), 'MNLI': ('test/data_for_tests/io/MNLI', MNLILoader, (5, 5, 5, 5, 6), True), + 'Quora': ('test/data_for_tests/io/Quora', QuoraLoader, (2, 2, 2), False), 'BQCorpus': ('test/data_for_tests/io/BQCorpus', BQCorpusLoader, (5, 5, 5), False), 'XNLI': ('test/data_for_tests/io/XNLI', XNLILoader, (6, 7, 6), False), 'LCQMC': ('test/data_for_tests/io/LCQMC', LCQMCLoader, (5, 6, 6), False), diff --git a/test/io/pipe/test_classification.py b/test/io/pipe/test_classification.py index be6127eb..036530c3 100644 --- a/test/io/pipe/test_classification.py +++ b/test/io/pipe/test_classification.py @@ -3,7 +3,7 @@ import os from fastNLP.io import DataBundle from fastNLP.io.pipe.classification import SSTPipe, SST2Pipe, IMDBPipe, YelpFullPipe, YelpPolarityPipe -from fastNLP.io.pipe.classification import ChnSentiCorpPipe +from fastNLP.io.pipe.classification import ChnSentiCorpPipe, THUCNewsPipe, WeiboSenti100kPipe @unittest.skipIf('TRAVIS' in os.environ, "Skip in travis") @@ -41,6 +41,8 @@ class TestRunClassificationPipe(unittest.TestCase): 'sst': ('test/data_for_tests/io/SST', SSTPipe, (6, 354, 6), (232, 5), False), 'imdb': ('test/data_for_tests/io/imdb', IMDBPipe, (6, 6, 6), (1670, 2), False), 'ChnSentiCorp': ('test/data_for_tests/io/ChnSentiCorp', ChnSentiCorpPipe, (6, 6, 6), (529, 1296, 1483, 2), False), + 'Chn-THUCNews': ('test/data_for_tests/io/THUCNews', THUCNewsPipe, (9, 9, 9), (1864, 9), False), + 'Chn-WeiboSenti100k': ('test/data_for_tests/io/WeiboSenti100k', WeiboSenti100kPipe, (7, 6, 6), (452, 2), False), } for k, v in data_set_dict.items(): path, pipe, data_set, vocab, warns = v diff --git a/test/io/pipe/test_matching.py b/test/io/pipe/test_matching.py index a901bc78..e337350a 100644 --- a/test/io/pipe/test_matching.py +++ b/test/io/pipe/test_matching.py @@ -3,9 +3,9 @@ import unittest import os from fastNLP.io import DataBundle -from fastNLP.io.pipe.matching import SNLIPipe, RTEPipe, QNLIPipe, MNLIPipe, \ +from fastNLP.io.pipe.matching import SNLIPipe, RTEPipe, QNLIPipe, QuoraPipe, MNLIPipe, \ XNLIPipe, BQCorpusPipe, LCQMCPipe -from fastNLP.io.pipe.matching import SNLIBertPipe, RTEBertPipe, QNLIBertPipe, MNLIBertPipe, \ +from fastNLP.io.pipe.matching import SNLIBertPipe, RTEBertPipe, QNLIBertPipe, QuoraBertPipe, MNLIBertPipe, \ XNLIBertPipe, BQCorpusBertPipe, LCQMCBertPipe @@ -75,7 +75,7 @@ class TestRunMatchingPipe(unittest.TestCase): def test_spacy(self): data_set_dict = { - 'RTE': ('test/data_for_tests/io/RTE', RTEPipe, RTEBertPipe, (5, 5, 5), (425, 2)), + 'Quora': ('test/data_for_tests/io/Quora', QuoraPipe, QuoraBertPipe, (2, 2, 2), (93, 2)), } for k, v in data_set_dict.items(): path, pipe1, pipe2, data_set, vocab = v