|
|
@@ -1,13 +1,12 @@ |
|
|
|
import os |
|
|
|
import configparser |
|
|
|
|
|
|
|
import json |
|
|
|
import os |
|
|
|
import unittest |
|
|
|
|
|
|
|
|
|
|
|
from fastNLP.loader.config_loader import ConfigSection, ConfigLoader |
|
|
|
from fastNLP.loader.dataset_loader import TokenizeDatasetLoader, POSDatasetLoader, LMDatasetLoader |
|
|
|
|
|
|
|
|
|
|
|
class TestConfigLoader(unittest.TestCase): |
|
|
|
def test_case_ConfigLoader(self): |
|
|
|
|
|
|
@@ -33,7 +32,7 @@ class TestConfigLoader(unittest.TestCase): |
|
|
|
return dict |
|
|
|
|
|
|
|
test_arg = ConfigSection() |
|
|
|
ConfigLoader("config", "").load_config(os.path.join("./test/loader", "config"), {"test": test_arg}) |
|
|
|
ConfigLoader("config").load_config(os.path.join("./test/loader", "config"), {"test": test_arg}) |
|
|
|
section = read_section_from_config(os.path.join("./test/loader", "config"), "test") |
|
|
|
|
|
|
|
for sec in section: |
|
|
@@ -54,18 +53,18 @@ class TestConfigLoader(unittest.TestCase): |
|
|
|
|
|
|
|
class TestDatasetLoader(unittest.TestCase): |
|
|
|
def test_case_TokenizeDatasetLoader(self): |
|
|
|
loader = TokenizeDatasetLoader("cws_pku_utf_8", "./test/data_for_tests/cws_pku_utf_8") |
|
|
|
loader = TokenizeDatasetLoader("./test/data_for_tests/cws_pku_utf_8") |
|
|
|
data = loader.load_pku(max_seq_len=32) |
|
|
|
print("pass TokenizeDatasetLoader test!") |
|
|
|
|
|
|
|
def test_case_POSDatasetLoader(self): |
|
|
|
loader = POSDatasetLoader("people", "./test/data_for_tests/people.txt") |
|
|
|
loader = POSDatasetLoader("./test/data_for_tests/people.txt") |
|
|
|
data = loader.load() |
|
|
|
datas = loader.load_lines() |
|
|
|
print("pass POSDatasetLoader test!") |
|
|
|
|
|
|
|
def test_case_LMDatasetLoader(self): |
|
|
|
loader = LMDatasetLoader("cws_pku_utf_8", "./test/data_for_tests/cws_pku_utf_8") |
|
|
|
loader = LMDatasetLoader("./test/data_for_tests/cws_pku_utf_8") |
|
|
|
data = loader.load() |
|
|
|
datas = loader.load_lines() |
|
|
|
print("pass TokenizeDatasetLoader test!") |
|
|
|
print("pass TokenizeDatasetLoader test!") |