|
|
@@ -64,13 +64,13 @@ class TestDatasetLoader(unittest.TestCase): |
|
|
|
def test_import(self): |
|
|
|
import fastNLP |
|
|
|
from fastNLP.io import SNLILoader |
|
|
|
ds = SNLILoader().process('./../data_for_tests/sample_snli.jsonl', to_lower=True, |
|
|
|
ds = SNLILoader().process('test/data_for_tests/sample_snli.jsonl', to_lower=True, |
|
|
|
get_index=True, seq_len_type='seq_len', extra_split=['-']) |
|
|
|
assert 'train' in ds.datasets |
|
|
|
assert len(ds.datasets) == 1 |
|
|
|
assert len(ds.datasets['train']) == 3 |
|
|
|
|
|
|
|
ds = SNLILoader().process('./../data_for_tests/sample_snli.jsonl', to_lower=True, |
|
|
|
ds = SNLILoader().process('test/data_for_tests/sample_snli.jsonl', to_lower=True, |
|
|
|
get_index=True, seq_len_type='seq_len') |
|
|
|
assert 'train' in ds.datasets |
|
|
|
assert len(ds.datasets) == 1 |
|
|
|