add test code and data for testing CHN NER and classification loader and pipe

test/data_for_tests/io/MSRA_NER/dev.conll

@@ -0,0 +1,38 @@
+把	O
+欧	B-LOC
+
+美	B-LOC
+、	O
+
+港	B-LOC
+台	B-LOC
+
+流	O
+行	O
+
+的	O
+食	O
+
+品	O
+类	O
+
+图	O
+谱	O
+
+马	B-PER
+列	B-PER
+
+主	O
+义	O
+
+在	O
+中	B-LOC
+
+国	I-LOC
+传	O
+
+播	O
+的	O
+
+历	O
+史	O

test/data_for_tests/io/MSRA_NER/test.conll

@@ -0,0 +1,31 @@
+中	B-ORG
+共	I-ORG
+
+中	I-ORG
+央	I-ORG
+
+致	O
+中	B-ORG
+
+国	I-ORG
+致	I-ORG
+
+公	I-ORG
+党	I-ORG
+
+十	I-ORG
+一	I-ORG
+
+大	I-ORG
+的	O
+
+贺	O
+词	O
+
+
+各	O
+
+位	O
+代	O
+
+表	O

test/data_for_tests/io/MSRA_NER/train.conll

@@ -0,0 +1,60 @@
+是	O
+我	O
+
+们	O
+收	O
+
+藏	O
+北	B-LOC
+
+京	I-LOC
+史	O
+
+料	O
+
+调	O
+查	O
+
+范	O
+围	O
+
+涉	O
+及	O
+
+故	B-LOC
+宫	I-LOC
+
+、	O
+历	B-LOC
+
+博	I-LOC
+、	O
+
+古	B-ORG
+研	I-ORG
+
+所	I-ORG
+、	O
+
+北	B-LOC
+大	I-LOC
+
+清	I-LOC
+华	I-LOC
+
+图	I-LOC
+书	I-LOC
+
+馆	I-LOC
+．	O
+
+夏	B-PER
+财	I-PER
+
+兴	I-PER
+家	O
+
+分	O
+到	O
+
+田	O

test/data_for_tests/io/peopledaily/dev.txt

@@ -0,0 +1,7 @@
+中 B-ORG
+共 I-ORG
+中 I-ORG
+央 I-ORG
+
+致 O
+中 B-ORG

test/data_for_tests/io/peopledaily/test.txt

@@ -0,0 +1,41 @@
+美 B-LOC
+国 I-LOC
+
+的 O
+华 B-PER
+
+莱 B-PER
+士 B-PER
+
+中 B-ORG
+共 I-ORG
+
+中 I-ORG
+央 I-ORG
+
+举 O
+办 O
+
+《 O
+“ O
+
+一 O
+国 O
+
+两 O
+制 O
+
+” O
+与 O
+
+香 B-LOC
+港 I-LOC
+
+基 O
+本 O
+
+法 O
+》 O
+
+讲 O
+座 O

test/data_for_tests/io/peopledaily/train.txt

@@ -0,0 +1,46 @@
+我 O
+们 O
+
+收 O
+藏 O
+
+北 B-LOC
+京 I-LOC
+
+史 O
+料 O
+
+历 B-LOC
+博 I-LOC
+
+、 O
+古 B-ORG
+研 I-ORG
+所 I-ORG
+
+、 O
+北 B-LOC
+
+大 I-LOC
+清 I-LOC
+
+华 I-LOC
+图 I-LOC
+
+书 I-LOC
+馆 I-LOC
+
+我 O
+们 O
+
+是 O
+受 O
+
+到 O
+郑 B-PER
+
+振 I-PER
+铎 I-PER
+
+先 O
+生 O

test/data_for_tests/io/weibo_NER/dev.conll

@@ -0,0 +1,21 @@
+老	B-PER.NOM
+百	I-PER.NOM
+姓	I-PER.NOM
+
+心	O
+
+新	B-GPE.NAM
+乡	I-GPE.NAM
+
+年	O
+
+大	B-ORG.NOM
+学	I-ORG.NOM
+
+同	O
+
+宿	B-LOC.NOM
+舍	I-LOC.NOM
+
+三	O
+年	O

test/data_for_tests/io/weibo_NER/test.conll

@@ -0,0 +1,17 @@
+感	O
+动	O
+
+了	O
+
+李	B-PER.NAM
+开	I-PER.NAM
+复	I-PER.NAM
+
+小	B-ORG.NOM
+学	I-ORG.NOM
+
+美	O
+术	O
+
+新	O
+课	O

test/data_for_tests/io/weibo_NER/train.conll

@@ -0,0 +1,69 @@
+坏      O
+男      B-PER.NOM
+人      I-PER.NOM
+
+男      B-PER.NOM
+人      I-PER.NOM
+帮      I-PER.NOM
+
+
+不      O
+
+南      B-GPE.NAM
+都      I-GPE.NAM
+
+南      B-GPE.NAM
+方      I-GPE.NAM
+都      I-GPE.NAM
+市      I-GPE.NAM
+
+的      O
+
+那      B-LOC.NOM
+座      I-LOC.NOM
+
+来      O
+
+学      B-ORG.NOM
+校      I-ORG.NOM
+
+的      O
+
+卫      B-ORG.NAM
+生      I-ORG.NAM
+部      I-ORG.NAM
+
+台      B-GPE.NAM
+灣      I-GPE.NAM
+
+火      B-LOC.NAM
+焰      I-LOC.NAM
+山      I-LOC.NAM
+
+的      O
+
+成       O
+李       B-PER.NAM
+力       I-PER.NAM
+帆       I-PER.NAM
+
+我       O
+
+南	B-GPE.NAM
+都	I-GPE.NAM
+
+深	B-GPE.NAM
+圳	I-GPE.NAM
+
+一	O
+个	O
+
+国	B-GPE.NOM
+家	I-GPE.NOM
+
+以	O
+
+民	B-PER.NOM
+
+为	O
+本	O

test/io/loader/test_classification_loader.py

@@ -31,6 +31,7 @@ class TestLoad(unittest.TestCase):
             'sst-2': ('test/data_for_tests/io/SST-2', SST2Loader, (5, 5, 5), True),
             'sst': ('test/data_for_tests/io/SST', SSTLoader, (6, 6, 6), False),
             'imdb': ('test/data_for_tests/io/imdb', IMDBLoader, (6, 6, 6), False),
+            'ChnSentiCorp': ('test/data_for_tests/io/ChnSentiCorp', ChnSentiCorpLoader, (6, 6, 6), False),
         }
         for k, v in data_set_dict.items():
             path, loader, data_set, warns = v

test/io/pipe/test_classification.py

@@ -40,15 +40,19 @@ class TestRunClassificationPipe(unittest.TestCase):
             'sst-2': ('test/data_for_tests/io/SST-2', SST2Pipe, (5, 5, 5), (139, 2), True),
             'sst': ('test/data_for_tests/io/SST', SSTPipe, (6, 354, 6), (232, 5), False),
             'imdb': ('test/data_for_tests/io/imdb', IMDBPipe, (6, 6, 6), (1670, 2), False),
+            'ChnSentiCorp': ('test/data_for_tests/io/ChnSentiCorp', ChnSentiCorpPipe, (6, 6, 6), (529, 1296, 1483, 2), False),
         }
         for k, v in data_set_dict.items():
             path, pipe, data_set, vocab, warns = v
             with self.subTest(pipe=pipe):
-                if warns:
-                    with self.assertWarns(Warning):
+                if 'Chn' not in k:
+                    if warns:
+                        with self.assertWarns(Warning):
+                            data_bundle = pipe(tokenizer='raw').process_from_file(path)
+                    else:
                         data_bundle = pipe(tokenizer='raw').process_from_file(path)
                 else:
-                    data_bundle = pipe(tokenizer='raw').process_from_file(path)
+                    data_bundle = pipe(bigrams=True, trigrams=True).process_from_file(path)
 
                 self.assertTrue(isinstance(data_bundle, DataBundle))
                 self.assertEqual(len(data_set), data_bundle.num_dataset)

test/io/pipe/test_conll.py

@@ -22,3 +22,19 @@ class TestRunPipe(unittest.TestCase):
                 print(pipe)
                 data_bundle = pipe().process_from_file('test/data_for_tests/conll_2003_example.txt')
                 print(data_bundle)
+
+
+class TestNERPipe(unittest.TestCase):
+    def test_process_from_file(self):
+        data_dict = {
+            'weibo_NER': WeiboNERPipe,
+            'peopledaily': PeopleDailyPipe,
+            'MSRA_NER': MsraNERPipe,
+        }
+        for k, v in data_dict.items():
+            pipe = v
+            with self.subTest(pipe=pipe):
+                data_bundle = pipe(bigrams=True, trigrams=True).process_from_file(f'test/data_for_tests/io/{k}')
+                print(data_bundle)
+                data_bundle = pipe(encoding_type='bioes').process_from_file(f'test/data_for_tests/io/{k}')
+                print(data_bundle)