From 51b493d7165cdb2045c2c8198217d23afb75d5eb Mon Sep 17 00:00:00 2001 From: ChenXin Date: Tue, 14 May 2019 23:21:21 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E6=94=B9=20io=20=E7=9A=84=E6=B5=8B?= =?UTF-8?q?=E8=AF=95=E6=96=87=E4=BB=B6;=20=E5=88=A0=E9=99=A4=E4=BA=86?= =?UTF-8?q?=E4=B8=80=E4=BA=9B=E8=BF=87=E6=97=B6=E7=9A=84=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/io/config | 62 ---------------- test/io/test_config_saver.py | 112 ----------------------------- test/io/test_dataset_loader.py | 10 +-- test/io/test_embed_loader.py | 17 +++-- test/modules/test_other_modules.py | 46 ------------ 5 files changed, 13 insertions(+), 234 deletions(-) delete mode 100644 test/io/config delete mode 100644 test/io/test_config_saver.py diff --git a/test/io/config b/test/io/config deleted file mode 100644 index 5ff9eacf..00000000 --- a/test/io/config +++ /dev/null @@ -1,62 +0,0 @@ -[test] -x = 1 - -y = 2 - -z = 3 -#this is an example -input = [1,2,3] - -text = "this is text" - -doubles = 0.8 - -tt = 0.5 - -test = 105 - -str = "this is a str" - -double = 0.5 - - -[t] -x = "this is an test section" - - - -[test-case-2] -double = 0.5 - -doubles = 0.8 - -tt = 0.5 - -test = 105 - -str = "this is a str" - -[another-test] -doubles = 0.8 - -tt = 0.5 - -test = 105 - -str = "this is a str" - -double = 0.5 - - -[one-another-test] -doubles = 0.8 - -tt = 0.5 - -test = 105 - -str = "this is a str" - -double = 0.5 - - diff --git a/test/io/test_config_saver.py b/test/io/test_config_saver.py deleted file mode 100644 index e5341d63..00000000 --- a/test/io/test_config_saver.py +++ /dev/null @@ -1,112 +0,0 @@ -import os -import unittest - -# from fastNLP.io import ConfigSection, ConfigLoader, ConfigSaver - - -class TestConfigSaver(unittest.TestCase): - def test_case_1(self): - config_file_dir = "." - config_file_name = "config" - config_file_path = os.path.join(config_file_dir, config_file_name) - - tmp_config_file_path = os.path.join(config_file_dir, "tmp_config") - - with open(config_file_path, "r") as f: - lines = f.readlines() - - standard_section = ConfigSection() - t_section = ConfigSection() - ConfigLoader().load_config(config_file_path, {"test": standard_section, "t": t_section}) - - config_saver = ConfigSaver(config_file_path) - - section = ConfigSection() - section["doubles"] = 0.8 - section["tt"] = 0.5 - section["test"] = 105 - section["str"] = "this is a str" - - test_case_2_section = section - test_case_2_section["double"] = 0.5 - - for k in section.__dict__.keys(): - standard_section[k] = section[k] - - config_saver.save_config_file("test", section) - config_saver.save_config_file("another-test", section) - config_saver.save_config_file("one-another-test", section) - config_saver.save_config_file("test-case-2", section) - - test_section = ConfigSection() - at_section = ConfigSection() - another_test_section = ConfigSection() - one_another_test_section = ConfigSection() - a_test_case_2_section = ConfigSection() - - ConfigLoader().load_config(config_file_path, {"test": test_section, - "another-test": another_test_section, - "t": at_section, - "one-another-test": one_another_test_section, - "test-case-2": a_test_case_2_section}) - - assert test_section == standard_section - assert at_section == t_section - assert another_test_section == section - assert one_another_test_section == section - assert a_test_case_2_section == test_case_2_section - - config_saver.save_config_file("test", section) - - with open(config_file_path, "w") as f: - f.writelines(lines) - - with open(tmp_config_file_path, "w") as f: - f.write('[test]\n') - f.write('this is an fault example\n') - - tmp_config_saver = ConfigSaver(tmp_config_file_path) - try: - tmp_config_saver._read_section() - except Exception as e: - pass - os.remove(tmp_config_file_path) - - try: - tmp_config_saver = ConfigSaver("file-NOT-exist") - except Exception as e: - pass - - def test_case_2(self): - config = "[section_A]\n[section_B]\n" - - with open("./test.cfg", "w", encoding="utf-8") as f: - f.write(config) - saver = ConfigSaver("./test.cfg") - - section = ConfigSection() - section["doubles"] = 0.8 - section["tt"] = [1, 2, 3] - section["test"] = 105 - section["str"] = "this is a str" - - saver.save_config_file("section_A", section) - - os.system("rm ./test.cfg") - - def test_case_3(self): - config = "[section_A]\ndoubles = 0.9\ntt = [1, 2, 3]\n[section_B]\n" - - with open("./test.cfg", "w", encoding="utf-8") as f: - f.write(config) - saver = ConfigSaver("./test.cfg") - - section = ConfigSection() - section["doubles"] = 0.8 - section["tt"] = [1, 2, 3] - section["test"] = 105 - section["str"] = "this is a str" - - saver.save_config_file("section_A", section) - - os.system("rm ./test.cfg") diff --git a/test/io/test_dataset_loader.py b/test/io/test_dataset_loader.py index 3c9d7c07..12d352b1 100644 --- a/test/io/test_dataset_loader.py +++ b/test/io/test_dataset_loader.py @@ -9,22 +9,22 @@ class TestDatasetLoader(unittest.TestCase): """ Test the the loader of Conll2003 dataset """ - dataset_path = "../data_for_tests/conll_2003_example.txt" + dataset_path = "test/data_for_tests/conll_2003_example.txt" loader = Conll2003Loader() dataset_2003 = loader.load(dataset_path) def test_PeopleDailyCorpusLoader(self): - data_set = PeopleDailyCorpusLoader().load("../data_for_tests/people_daily_raw.txt") + data_set = PeopleDailyCorpusLoader().load("test/data_for_tests/people_daily_raw.txt") def test_CSVLoader(self): ds = CSVLoader(sep='\t', headers=['words', 'label']) \ - .load('../data_for_tests/tutorial_sample_dataset.csv') + .load('test/data_for_tests/tutorial_sample_dataset.csv') assert len(ds) > 0 def test_SNLILoader(self): - ds = SNLILoader().load('../data_for_tests/sample_snli.jsonl') + ds = SNLILoader().load('test/data_for_tests/sample_snli.jsonl') assert len(ds) == 3 def test_JsonLoader(self): - ds = JsonLoader().load('../data_for_tests/sample_snli.jsonl') + ds = JsonLoader().load('test/data_for_tests/sample_snli.jsonl') assert len(ds) == 3 diff --git a/test/io/test_embed_loader.py b/test/io/test_embed_loader.py index d43a00fe..ff8ecfcf 100644 --- a/test/io/test_embed_loader.py +++ b/test/io/test_embed_loader.py @@ -3,15 +3,13 @@ import numpy as np from fastNLP import Vocabulary from fastNLP.io import EmbedLoader -import os -from fastNLP.io.dataset_loader import SSTLoader -from fastNLP.core.const import Const as C + class TestEmbedLoader(unittest.TestCase): def test_load_with_vocab(self): vocab = Vocabulary() - glove = "../data_for_tests/glove.6B.50d_test.txt" - word2vec = "../data_for_tests/word2vec_test.txt" + glove = "test/data_for_tests/glove.6B.50d_test.txt" + word2vec = "test/data_for_tests/word2vec_test.txt" vocab.add_word('the') vocab.add_word('none') g_m = EmbedLoader.load_with_vocab(glove, vocab) @@ -19,11 +17,11 @@ class TestEmbedLoader(unittest.TestCase): w_m = EmbedLoader.load_with_vocab(word2vec, vocab, normalize=True) self.assertEqual(w_m.shape, (4, 50)) self.assertAlmostEqual(np.linalg.norm(w_m, axis=1).sum(), 4) - + def test_load_without_vocab(self): words = ['the', 'of', 'in', 'a', 'to', 'and'] - glove = "../data_for_tests/glove.6B.50d_test.txt" - word2vec = "../data_for_tests/word2vec_test.txt" + glove = "test/data_for_tests/glove.6B.50d_test.txt" + word2vec = "test/data_for_tests/word2vec_test.txt" g_m, vocab = EmbedLoader.load_without_vocab(glove) self.assertEqual(g_m.shape, (8, 50)) for word in words: @@ -39,9 +37,10 @@ class TestEmbedLoader(unittest.TestCase): self.assertAlmostEqual(np.linalg.norm(w_m, axis=1).sum(), 7) for word in words: self.assertIn(word, vocab) - + def test_read_all_glove(self): pass + # TODO # 这是可以运行的，但是总数少于行数，应该是由于glove有重复的word # path = '/where/to/read/full/glove' # init_embed, vocab = EmbedLoader.load_without_vocab(path, error='strict') diff --git a/test/modules/test_other_modules.py b/test/modules/test_other_modules.py index ef5020c1..c5462623 100644 --- a/test/modules/test_other_modules.py +++ b/test/modules/test_other_modules.py @@ -2,55 +2,9 @@ import unittest import torch -# from fastNLP.modules.other_modules import GroupNorm, LayerNormalization, BiLinear, BiAffine from fastNLP.modules.encoder.star_transformer import StarTransformer -class TestGroupNorm(unittest.TestCase): - def test_case_1(self): - gn = GroupNorm(num_features=1, num_groups=10, eps=1.5e-5) - x = torch.randn((20, 50, 10)) - y = gn(x) - - -class TestLayerNormalization(unittest.TestCase): - def test_case_1(self): - ln = LayerNormalization(layer_size=5, eps=2e-3) - x = torch.randn((20, 50, 5)) - y = ln(x) - - -class TestBiLinear(unittest.TestCase): - def test_case_1(self): - bl = BiLinear(n_left=5, n_right=5, n_out=10, bias=True) - x_left = torch.randn((7, 10, 20, 5)) - x_right = torch.randn((7, 10, 20, 5)) - y = bl(x_left, x_right) - print(bl) - bl2 = BiLinear(n_left=15, n_right=15, n_out=10, bias=True) - - -class TestBiAffine(unittest.TestCase): - def test_case_1(self): - batch_size = 16 - encoder_length = 21 - decoder_length = 32 - layer = BiAffine(10, 10, 25, biaffine=True) - decoder_input = torch.randn((batch_size, encoder_length, 10)) - encoder_input = torch.randn((batch_size, decoder_length, 10)) - y = layer(decoder_input, encoder_input) - self.assertEqual(tuple(y.shape), (batch_size, 25, encoder_length, decoder_length)) - - def test_case_2(self): - batch_size = 16 - encoder_length = 21 - decoder_length = 32 - layer = BiAffine(10, 10, 25, biaffine=False) - decoder_input = torch.randn((batch_size, encoder_length, 10)) - encoder_input = torch.randn((batch_size, decoder_length, 10)) - y = layer(decoder_input, encoder_input) - self.assertEqual(tuple(y.shape), (batch_size, 25, encoder_length, 1)) - class TestStarTransformer(unittest.TestCase): def test_1(self): model = StarTransformer(num_layers=6, hidden_size=100, num_head=8, head_dim=20, max_len=100)