| @@ -1,13 +1,12 @@ | |||||
| import os | |||||
| import configparser | import configparser | ||||
| import json | import json | ||||
| import os | |||||
| import unittest | import unittest | ||||
| from fastNLP.loader.config_loader import ConfigSection, ConfigLoader | from fastNLP.loader.config_loader import ConfigSection, ConfigLoader | ||||
| from fastNLP.loader.dataset_loader import TokenizeDatasetLoader, POSDatasetLoader, LMDatasetLoader | from fastNLP.loader.dataset_loader import TokenizeDatasetLoader, POSDatasetLoader, LMDatasetLoader | ||||
| class TestConfigLoader(unittest.TestCase): | class TestConfigLoader(unittest.TestCase): | ||||
| def test_case_ConfigLoader(self): | def test_case_ConfigLoader(self): | ||||
| @@ -33,7 +32,7 @@ class TestConfigLoader(unittest.TestCase): | |||||
| return dict | return dict | ||||
| test_arg = ConfigSection() | test_arg = ConfigSection() | ||||
| ConfigLoader("config", "").load_config(os.path.join("./test/loader", "config"), {"test": test_arg}) | |||||
| ConfigLoader("config").load_config(os.path.join("./test/loader", "config"), {"test": test_arg}) | |||||
| section = read_section_from_config(os.path.join("./test/loader", "config"), "test") | section = read_section_from_config(os.path.join("./test/loader", "config"), "test") | ||||
| for sec in section: | for sec in section: | ||||
| @@ -54,18 +53,18 @@ class TestConfigLoader(unittest.TestCase): | |||||
| class TestDatasetLoader(unittest.TestCase): | class TestDatasetLoader(unittest.TestCase): | ||||
| def test_case_TokenizeDatasetLoader(self): | def test_case_TokenizeDatasetLoader(self): | ||||
| loader = TokenizeDatasetLoader("cws_pku_utf_8", "./test/data_for_tests/cws_pku_utf_8") | |||||
| loader = TokenizeDatasetLoader("./test/data_for_tests/cws_pku_utf_8") | |||||
| data = loader.load_pku(max_seq_len=32) | data = loader.load_pku(max_seq_len=32) | ||||
| print("pass TokenizeDatasetLoader test!") | print("pass TokenizeDatasetLoader test!") | ||||
| def test_case_POSDatasetLoader(self): | def test_case_POSDatasetLoader(self): | ||||
| loader = POSDatasetLoader("people", "./test/data_for_tests/people.txt") | |||||
| loader = POSDatasetLoader("./test/data_for_tests/people.txt") | |||||
| data = loader.load() | data = loader.load() | ||||
| datas = loader.load_lines() | datas = loader.load_lines() | ||||
| print("pass POSDatasetLoader test!") | print("pass POSDatasetLoader test!") | ||||
| def test_case_LMDatasetLoader(self): | def test_case_LMDatasetLoader(self): | ||||
| loader = LMDatasetLoader("cws_pku_utf_8", "./test/data_for_tests/cws_pku_utf_8") | |||||
| loader = LMDatasetLoader("./test/data_for_tests/cws_pku_utf_8") | |||||
| data = loader.load() | data = loader.load() | ||||
| datas = loader.load_lines() | datas = loader.load_lines() | ||||
| print("pass TokenizeDatasetLoader test!") | |||||
| print("pass TokenizeDatasetLoader test!") | |||||