+
+ | fastNLP |
+ an open-source NLP library |
+
+
+ | fastNLP.core |
+ trainer, tester, predictor |
+
+
+ | fastNLP.loader |
+ all kinds of loaders/readers |
+
+
+ | fastNLP.models |
+ a collection of NLP models |
+
+
+ | fastNLP.modules |
+ a collection of PyTorch sub-models/components/wheels |
+
+
+ | fastNLP.saver |
+ all kinds of savers/writers |
+
+
+ | fastNLP.fastnlp |
+ a high-level interface for prediction |
+
+
\ No newline at end of file
diff --git a/docs/source/user/quickstart.rst b/docs/source/user/quickstart.rst
index 21f0855f..24c7363c 100644
--- a/docs/source/user/quickstart.rst
+++ b/docs/source/user/quickstart.rst
@@ -18,7 +18,7 @@ pre-processing data, constructing model and training model.
from fastNLP.modules import aggregation
from fastNLP.modules import decoder
- from fastNLP.loader.dataset_loader import ClassDatasetLoader
+ from fastNLP.loader.dataset_loader import ClassDataSetLoader
from fastNLP.loader.preprocess import ClassPreprocess
from fastNLP.core.trainer import ClassificationTrainer
from fastNLP.core.inference import ClassificationInfer
@@ -50,7 +50,7 @@ pre-processing data, constructing model and training model.
train_path = 'test/data_for_tests/text_classify.txt' # training set file
# load dataset
- ds_loader = ClassDatasetLoader("train", train_path)
+ ds_loader = ClassDataSetLoader("train", train_path)
data = ds_loader.load()
# pre-process dataset
diff --git a/examples/readme_example.py b/examples/readme_example.py
index 74e20c57..9da2787b 100644
--- a/examples/readme_example.py
+++ b/examples/readme_example.py
@@ -3,7 +3,7 @@ from fastNLP.core.optimizer import Optimizer
from fastNLP.core.predictor import ClassificationInfer
from fastNLP.core.preprocess import ClassPreprocess
from fastNLP.core.trainer import ClassificationTrainer
-from fastNLP.loader.dataset_loader import ClassDatasetLoader
+from fastNLP.loader.dataset_loader import ClassDataSetLoader
from fastNLP.models.base_model import BaseModel
from fastNLP.modules import aggregator
from fastNLP.modules import decoder
@@ -36,7 +36,7 @@ data_dir = 'save/' # directory to save data and model
train_path = './data_for_tests/text_classify.txt' # training set file
# load dataset
-ds_loader = ClassDatasetLoader(train_path)
+ds_loader = ClassDataSetLoader()
data = ds_loader.load()
# pre-process dataset
diff --git a/fastNLP/core/batch.py b/fastNLP/core/batch.py
index 98d7c8da..bf837d0f 100644
--- a/fastNLP/core/batch.py
+++ b/fastNLP/core/batch.py
@@ -17,7 +17,7 @@ class Batch(object):
:param dataset: a DataSet object
:param batch_size: int, the size of the batch
:param sampler: a Sampler object
- :param use_cuda: bool, whetjher to use GPU
+ :param use_cuda: bool, whether to use GPU
"""
self.dataset = dataset
@@ -37,15 +37,12 @@ class Batch(object):
"""
:return batch_x: dict of (str: torch.LongTensor), which means (field name: tensor of shape [batch_size, padding_length])
- batch_x also contains an item (str: list of int) about origin lengths,
- which means ("field_name_origin_len": origin lengths).
E.g.
::
{'text': tensor([[ 0, 1, 2, 3, 0, 0, 0], 4, 5, 2, 6, 7, 8, 9]]), 'text_origin_len': [4, 7]})
batch_y: dict of (str: torch.LongTensor), which means (field name: tensor of shape [batch_size, padding_length])
All tensors in both batch_x and batch_y will be cuda tensors if use_cuda is True.
- The names of fields are defined in preprocessor's convert_to_dataset method.
"""
if self.curidx >= len(self.idx_list):
@@ -54,10 +51,9 @@ class Batch(object):
endidx = min(self.curidx + self.batch_size, len(self.idx_list))
padding_length = {field_name: max(field_length[self.curidx: endidx])
for field_name, field_length in self.lengths.items()}
- origin_lengths = {field_name: field_length[self.curidx: endidx]
- for field_name, field_length in self.lengths.items()}
-
batch_x, batch_y = defaultdict(list), defaultdict(list)
+
+ # transform index to tensor and do padding for sequences
for idx in range(self.curidx, endidx):
x, y = self.dataset.to_tensor(idx, padding_length)
for name, tensor in x.items():
@@ -65,8 +61,7 @@ class Batch(object):
for name, tensor in y.items():
batch_y[name].append(tensor)
- batch_origin_length = {}
- # combine instances into a batch
+ # combine instances to form a batch
for batch in (batch_x, batch_y):
for name, tensor_list in batch.items():
if self.use_cuda:
@@ -74,14 +69,6 @@ class Batch(object):
else:
batch[name] = torch.stack(tensor_list, dim=0)
- # add origin lengths in batch_x
- for name, tensor in batch_x.items():
- if self.use_cuda:
- batch_origin_length[name + "_origin_len"] = torch.LongTensor(origin_lengths[name]).cuda()
- else:
- batch_origin_length[name + "_origin_len"] = torch.LongTensor(origin_lengths[name])
- batch_x.update(batch_origin_length)
-
self.curidx = endidx
return batch_x, batch_y
diff --git a/fastNLP/core/dataset.py b/fastNLP/core/dataset.py
index bb1a1890..13370969 100644
--- a/fastNLP/core/dataset.py
+++ b/fastNLP/core/dataset.py
@@ -1,7 +1,12 @@
+import random
+import sys
from collections import defaultdict
+from copy import deepcopy
-from fastNLP.core.field import TextField
+from fastNLP.core.field import TextField, LabelField
from fastNLP.core.instance import Instance
+from fastNLP.core.vocabulary import Vocabulary
+from fastNLP.loader.dataset_loader import POSDataSetLoader, ClassDataSetLoader
def create_dataset_from_lists(str_lists: list, word_vocab: dict, has_target: bool = False, label_vocab: dict = None):
@@ -65,17 +70,19 @@ class DataSet(list):
"""A DataSet object is a list of Instance objects.
"""
- def __init__(self, name="", instances=None):
+
+ def __init__(self, name="", instances=None, load_func=None):
"""
:param name: str, the name of the dataset. (default: "")
:param instances: list of Instance objects. (default: None)
-
+ :param load_func: a function that takes the dataset path (string) as input and returns multi-level lists.
"""
list.__init__([])
self.name = name
if instances is not None:
self.extend(instances)
+ self.data_set_load_func = load_func
def index_all(self, vocab):
for ins in self:
@@ -109,3 +116,191 @@ class DataSet(list):
for field_name, field_length in ins.get_length().items():
lengths[field_name].append(field_length)
return lengths
+
+ def convert(self, data):
+ """Convert lists of strings into Instances with Fields, creating Vocabulary for labeled data. Used in Training."""
+ raise NotImplementedError
+
+ def convert_with_vocabs(self, data, vocabs):
+ """Convert lists of strings into Instances with Fields, using existing Vocabulary, with labels. Used in Testing."""
+ raise NotImplementedError
+
+ def convert_for_infer(self, data, vocabs):
+ """Convert lists of strings into Instances with Fields, using existing Vocabulary, without labels. Used in predicting."""
+
+ def load(self, data_path, vocabs=None, infer=False):
+ """Load data from the given files.
+
+ :param data_path: str, the path to the data
+ :param infer: bool. If True, there is no label information in the data. Default: False.
+ :param vocabs: dict of (name: Vocabulary object), used to index data. If not provided, a new vocabulary will be constructed.
+
+ """
+ raw_data = self.data_set_load_func(data_path)
+ if infer is True:
+ self.convert_for_infer(raw_data, vocabs)
+ else:
+ if vocabs is not None:
+ self.convert_with_vocabs(raw_data, vocabs)
+ else:
+ self.convert(raw_data)
+
+ def load_raw(self, raw_data, vocabs):
+ """Load raw data without loader. Used in FastNLP class.
+
+ :param raw_data:
+ :param vocabs:
+ :return:
+ """
+ self.convert_for_infer(raw_data, vocabs)
+
+ def split(self, ratio, shuffle=True):
+ """Train/dev splitting
+
+ :param ratio: float, between 0 and 1. The ratio of development set in origin data set.
+ :param shuffle: bool, whether shuffle the data set before splitting. Default: True.
+ :return train_set: a DataSet object, representing the training set
+ dev_set: a DataSet object, representing the validation set
+
+ """
+ assert 0 < ratio < 1
+ if shuffle:
+ random.shuffle(self)
+ split_idx = int(len(self) * ratio)
+ dev_set = deepcopy(self)
+ train_set = deepcopy(self)
+ del train_set[:split_idx]
+ del dev_set[split_idx:]
+ return train_set, dev_set
+
+
+class SeqLabelDataSet(DataSet):
+ def __init__(self, instances=None, load_func=POSDataSetLoader().load):
+ super(SeqLabelDataSet, self).__init__(name="", instances=instances, load_func=load_func)
+ self.word_vocab = Vocabulary()
+ self.label_vocab = Vocabulary()
+
+ def convert(self, data):
+ """Convert lists of strings into Instances with Fields.
+
+ :param data: 3-level lists. Entries are strings.
+ """
+ bar = ProgressBar(total=len(data))
+ for example in data:
+ word_seq, label_seq = example[0], example[1]
+ # list, list
+ self.word_vocab.update(word_seq)
+ self.label_vocab.update(label_seq)
+ x = TextField(word_seq, is_target=False)
+ x_len = LabelField(len(word_seq), is_target=False)
+ y = TextField(label_seq, is_target=False)
+ instance = Instance()
+ instance.add_field("word_seq", x)
+ instance.add_field("truth", y)
+ instance.add_field("word_seq_origin_len", x_len)
+ self.append(instance)
+ bar.move()
+ self.index_field("word_seq", self.word_vocab)
+ self.index_field("truth", self.label_vocab)
+ # no need to index "word_seq_origin_len"
+
+ def convert_with_vocabs(self, data, vocabs):
+ for example in data:
+ word_seq, label_seq = example[0], example[1]
+ # list, list
+ x = TextField(word_seq, is_target=False)
+ x_len = LabelField(len(word_seq), is_target=False)
+ y = TextField(label_seq, is_target=False)
+ instance = Instance()
+ instance.add_field("word_seq", x)
+ instance.add_field("truth", y)
+ instance.add_field("word_seq_origin_len", x_len)
+ self.append(instance)
+ self.index_field("word_seq", vocabs["word_vocab"])
+ self.index_field("truth", vocabs["label_vocab"])
+ # no need to index "word_seq_origin_len"
+
+ def convert_for_infer(self, data, vocabs):
+ for word_seq in data:
+ # list
+ x = TextField(word_seq, is_target=False)
+ x_len = LabelField(len(word_seq), is_target=False)
+ instance = Instance()
+ instance.add_field("word_seq", x)
+ instance.add_field("word_seq_origin_len", x_len)
+ self.append(instance)
+ self.index_field("word_seq", vocabs["word_vocab"])
+ # no need to index "word_seq_origin_len"
+
+
+class TextClassifyDataSet(DataSet):
+ def __init__(self, instances=None, load_func=ClassDataSetLoader().load):
+ super(TextClassifyDataSet, self).__init__(name="", instances=instances, load_func=load_func)
+ self.word_vocab = Vocabulary()
+ self.label_vocab = Vocabulary(need_default=False)
+
+ def convert(self, data):
+ for example in data:
+ word_seq, label = example[0], example[1]
+ # list, str
+ self.word_vocab.update(word_seq)
+ self.label_vocab.update(label)
+ x = TextField(word_seq, is_target=False)
+ y = LabelField(label, is_target=True)
+ instance = Instance()
+ instance.add_field("word_seq", x)
+ instance.add_field("label", y)
+ self.append(instance)
+ self.index_field("word_seq", self.word_vocab)
+ self.index_field("label", self.label_vocab)
+
+ def convert_with_vocabs(self, data, vocabs):
+ for example in data:
+ word_seq, label = example[0], example[1]
+ # list, str
+ x = TextField(word_seq, is_target=False)
+ y = LabelField(label, is_target=True)
+ instance = Instance()
+ instance.add_field("word_seq", x)
+ instance.add_field("label", y)
+ self.append(instance)
+ self.index_field("word_seq", vocabs["word_vocab"])
+ self.index_field("label", vocabs["label_vocab"])
+
+ def convert_for_infer(self, data, vocabs):
+ for word_seq in data:
+ # list
+ x = TextField(word_seq, is_target=False)
+ instance = Instance()
+ instance.add_field("word_seq", x)
+ self.append(instance)
+ self.index_field("word_seq", vocabs["word_vocab"])
+
+
+def change_field_is_target(data_set, field_name, new_target):
+ """Change the flag of is_target in a field.
+
+ :param data_set: a DataSet object
+ :param field_name: str, the name of the field
+ :param new_target: one of (True, False, None), representing this field is batch_x / is batch_y / neither.
+
+ """
+ for inst in data_set:
+ inst.fields[field_name].is_target = new_target
+
+
+class ProgressBar:
+
+ def __init__(self, count=0, total=0, width=100):
+ self.count = count
+ self.total = total
+ self.width = width
+
+ def move(self):
+ self.count += 1
+ progress = self.width * self.count // self.total
+ sys.stdout.write('{0:3}/{1:3}: '.format(self.count, self.total))
+ sys.stdout.write('#' * progress + '-' * (self.width - progress) + '\r')
+ if progress == self.width:
+ sys.stdout.write('\n')
+ sys.stdout.flush()
diff --git a/fastNLP/core/field.py b/fastNLP/core/field.py
index f5347bd6..b57b9bb6 100644
--- a/fastNLP/core/field.py
+++ b/fastNLP/core/field.py
@@ -59,6 +59,9 @@ class TextField(Field):
class LabelField(Field):
+ """The Field representing a single label. Can be a string or integer.
+
+ """
def __init__(self, label, is_target=True):
super(LabelField, self).__init__(is_target)
self.label = label
@@ -73,13 +76,14 @@ class LabelField(Field):
def index(self, vocab):
if self._index is None:
- self._index = vocab[self.label]
+ if isinstance(self.label, str):
+ self._index = vocab[self.label]
return self._index
def to_tensor(self, padding_length):
if self._index is None:
if isinstance(self.label, int):
- return torch.LongTensor([self.label])
+ return torch.tensor(self.label)
elif isinstance(self.label, str):
raise RuntimeError("Field {} not indexed. Call index method.".format(self.label))
else:
diff --git a/fastNLP/core/instance.py b/fastNLP/core/instance.py
index 32f95197..ebf01912 100644
--- a/fastNLP/core/instance.py
+++ b/fastNLP/core/instance.py
@@ -46,8 +46,11 @@ class Instance(object):
tensor_x = {}
tensor_y = {}
for name, field in self.fields.items():
- if field.is_target:
+ if field.is_target is True:
tensor_y[name] = field.to_tensor(padding_length[name])
- else:
+ elif field.is_target is False:
tensor_x[name] = field.to_tensor(padding_length[name])
+ else:
+ # is_target is None
+ continue
return tensor_x, tensor_y
diff --git a/fastNLP/core/loss.py b/fastNLP/core/loss.py
index 8a0eedd7..16b5eac2 100644
--- a/fastNLP/core/loss.py
+++ b/fastNLP/core/loss.py
@@ -33,10 +33,25 @@ class Loss(object):
"""Given a name of a loss function, return it from PyTorch.
:param loss_name: str, the name of a loss function
+
+ - cross_entropy: combines log softmax and nll loss in a single function.
+ - nll: negative log likelihood
+
:return loss: a PyTorch loss
"""
+
+ class InnerCrossEntropy:
+ """A simple wrapper to guarantee input shapes."""
+
+ def __init__(self):
+ self.f = torch.nn.CrossEntropyLoss()
+
+ def __call__(self, predict, truth):
+ truth = truth.view(-1, )
+ return self.f(predict, truth)
+
if loss_name == "cross_entropy":
- return torch.nn.CrossEntropyLoss()
+ return InnerCrossEntropy()
elif loss_name == 'nll':
return torch.nn.NLLLoss()
else:
diff --git a/fastNLP/core/metrics.py b/fastNLP/core/metrics.py
index 7bf4b034..6eedd214 100644
--- a/fastNLP/core/metrics.py
+++ b/fastNLP/core/metrics.py
@@ -4,6 +4,59 @@ import numpy as np
import torch
+class Evaluator(object):
+ def __init__(self):
+ pass
+
+ def __call__(self, predict, truth):
+ """
+
+ :param predict: list of tensors, the network outputs from all batches.
+ :param truth: list of dict, the ground truths from all batch_y.
+ :return:
+ """
+ raise NotImplementedError
+
+
+class ClassifyEvaluator(Evaluator):
+ def __init__(self):
+ super(ClassifyEvaluator, self).__init__()
+
+ def __call__(self, predict, truth):
+ y_prob = [torch.nn.functional.softmax(y_logit, dim=-1) for y_logit in predict]
+ y_prob = torch.cat(y_prob, dim=0)
+ y_pred = torch.argmax(y_prob, dim=-1)
+ y_true = torch.cat(truth, dim=0)
+ acc = float(torch.sum(y_pred == y_true)) / len(y_true)
+ return {"accuracy": acc}
+
+
+class SeqLabelEvaluator(Evaluator):
+ def __init__(self):
+ super(SeqLabelEvaluator, self).__init__()
+
+ def __call__(self, predict, truth):
+ """
+
+ :param predict: list of List, the network outputs from all batches.
+ :param truth: list of dict, the ground truths from all batch_y.
+ :return accuracy:
+ """
+ truth = [item["truth"] for item in truth]
+ total_correct, total_count= 0., 0.
+ for x, y in zip(predict, truth):
+ x = torch.Tensor(x)
+ y = y.to(x) # make sure they are in the same device
+ mask = x.ge(1).float()
+ # correct = torch.sum(x * mask.float() == (y * mask.long()).float())
+ correct = torch.sum(x * mask == y * mask)
+ correct -= torch.sum(x.le(0))
+ total_correct += float(correct)
+ total_count += float(torch.sum(mask))
+ accuracy = total_correct / total_count
+ return {"accuracy": float(accuracy)}
+
+
def _conver_numpy(x):
"""convert input data to numpy array
diff --git a/fastNLP/core/predictor.py b/fastNLP/core/predictor.py
index 6bbb1bee..14c4e8c1 100644
--- a/fastNLP/core/predictor.py
+++ b/fastNLP/core/predictor.py
@@ -16,43 +16,42 @@ class Predictor(object):
Currently, Predictor does not support GPU.
"""
- def __init__(self, pickle_path, task):
+ def __init__(self, pickle_path, post_processor):
"""
:param pickle_path: str, the path to the pickle files.
- :param task: str, specify which task the predictor will perform. One of ("seq_label", "text_classify").
+ :param post_processor: a function or callable object, that takes list of batch outputs as input
"""
self.batch_size = 1
self.batch_output = []
self.pickle_path = pickle_path
- self._task = task # one of ("seq_label", "text_classify")
- self.label_vocab = load_pickle(self.pickle_path, "class2id.pkl")
+ self._post_processor = post_processor
+ self.label_vocab = load_pickle(self.pickle_path, "label2id.pkl")
self.word_vocab = load_pickle(self.pickle_path, "word2id.pkl")
def predict(self, network, data):
"""Perform inference using the trained model.
:param network: a PyTorch model (cpu)
- :param data: list of list of strings, [num_examples, seq_len]
+ :param data: a DataSet object.
:return: list of list of strings, [num_examples, tag_seq_length]
"""
# transform strings into DataSet object
- data = self.prepare_input(data)
+ # data = self.prepare_input(data)
# turn on the testing mode; clean up the history
self.mode(network, test=True)
- self.batch_output.clear()
+ batch_output = []
data_iterator = Batch(data, batch_size=self.batch_size, sampler=SequentialSampler(), use_cuda=False)
for batch_x, _ in data_iterator:
with torch.no_grad():
prediction = self.data_forward(network, batch_x)
+ batch_output.append(prediction)
- self.batch_output.append(prediction)
-
- return self.prepare_output(self.batch_output)
+ return self._post_processor(batch_output, self.label_vocab)
def mode(self, network, test=True):
if test:
@@ -62,13 +61,7 @@ class Predictor(object):
def data_forward(self, network, x):
"""Forward through network."""
- if self._task == "seq_label":
- y = network(x["word_seq"], x["word_seq_origin_len"])
- y = network.prediction(y)
- elif self._task == "text_classify":
- y = network(x["word_seq"])
- else:
- raise NotImplementedError("Unknown task type {}.".format(self._task))
+ y = network(**x)
return y
def prepare_input(self, data):
@@ -88,39 +81,32 @@ class Predictor(object):
assert isinstance(data, list)
return create_dataset_from_lists(data, self.word_vocab, has_target=False)
- def prepare_output(self, data):
- """Transform list of batch outputs into strings."""
- if self._task == "seq_label":
- return self._seq_label_prepare_output(data)
- elif self._task == "text_classify":
- return self._text_classify_prepare_output(data)
- else:
- raise NotImplementedError("Unknown task type {}".format(self._task))
-
- def _seq_label_prepare_output(self, batch_outputs):
- results = []
- for batch in batch_outputs:
- for example in np.array(batch):
- results.append([self.label_vocab.to_word(int(x)) for x in example])
- return results
-
- def _text_classify_prepare_output(self, batch_outputs):
- results = []
- for batch_out in batch_outputs:
- idx = np.argmax(batch_out.detach().numpy(), axis=-1)
- results.extend([self.label_vocab.to_word(i) for i in idx])
- return results
-
class SeqLabelInfer(Predictor):
def __init__(self, pickle_path):
print(
- "[FastNLP Warning] SeqLabelInfer will be deprecated. Please use Predictor with argument 'task'='seq_label'.")
- super(SeqLabelInfer, self).__init__(pickle_path, "seq_label")
+ "[FastNLP Warning] SeqLabelInfer will be deprecated. Please use Predictor directly.")
+ super(SeqLabelInfer, self).__init__(pickle_path, seq_label_post_processor)
class ClassificationInfer(Predictor):
def __init__(self, pickle_path):
print(
- "[FastNLP Warning] ClassificationInfer will be deprecated. Please use Predictor with argument 'task'='text_classify'.")
- super(ClassificationInfer, self).__init__(pickle_path, "text_classify")
+ "[FastNLP Warning] ClassificationInfer will be deprecated. Please use Predictor directly.")
+ super(ClassificationInfer, self).__init__(pickle_path, text_classify_post_processor)
+
+
+def seq_label_post_processor(batch_outputs, label_vocab):
+ results = []
+ for batch in batch_outputs:
+ for example in np.array(batch):
+ results.append([label_vocab.to_word(int(x)) for x in example])
+ return results
+
+
+def text_classify_post_processor(batch_outputs, label_vocab):
+ results = []
+ for batch_out in batch_outputs:
+ idx = np.argmax(batch_out.detach().numpy(), axis=-1)
+ results.extend([label_vocab.to_word(i) for i in idx])
+ return results
diff --git a/fastNLP/core/preprocess.py b/fastNLP/core/preprocess.py
index 619d3967..5e77649e 100644
--- a/fastNLP/core/preprocess.py
+++ b/fastNLP/core/preprocess.py
@@ -18,6 +18,9 @@ def save_pickle(obj, pickle_path, file_name):
:param pickle_path: str, the directory where the pickle file is to be saved
:param file_name: str, the name of the pickle file. In general, it should be ended by "pkl".
"""
+ if not os.path.exists(pickle_path):
+ os.mkdir(pickle_path)
+ print("make dir {} before saving pickle file".format(pickle_path))
with open(os.path.join(pickle_path, file_name), "wb") as f:
_pickle.dump(obj, f)
print("{} saved in {}".format(file_name, pickle_path))
@@ -66,14 +69,27 @@ class Preprocessor(object):
Preprocessors will check if those files are already in the directory and will reuse them in future calls.
"""
- def __init__(self, label_is_seq=False):
+ def __init__(self, label_is_seq=False, share_vocab=False, add_char_field=False):
"""
:param label_is_seq: bool, whether label is a sequence. If True, label vocabulary will preserve
several special tokens for sequence processing.
+ :param share_vocab: bool, whether word sequence and label sequence share the same vocabulary. Typically, this
+ is only available when label_is_seq is True. Default: False.
+ :param add_char_field: bool, whether to add character representations to all TextFields. Default: False.
"""
+ print("Preprocessor is about to deprecate. Please use DataSet class.")
self.data_vocab = Vocabulary()
- self.label_vocab = Vocabulary(need_default=label_is_seq)
+ if label_is_seq is True:
+ if share_vocab is True:
+ self.label_vocab = self.data_vocab
+ else:
+ self.label_vocab = Vocabulary()
+ else:
+ self.label_vocab = Vocabulary(need_default=False)
+
+ self.character_vocab = Vocabulary(need_default=False)
+ self.add_char_field = add_char_field
@property
def vocab_size(self):
@@ -83,6 +99,12 @@ class Preprocessor(object):
def num_classes(self):
return len(self.label_vocab)
+ @property
+ def char_vocab_size(self):
+ if self.character_vocab is None:
+ self.build_char_dict()
+ return len(self.character_vocab)
+
def run(self, train_dev_data, test_data=None, pickle_path="./", train_dev_split=0, cross_val=False, n_fold=10):
"""Main pre-processing pipeline.
@@ -96,7 +118,6 @@ class Preprocessor(object):
If train_dev_split > 0, return one more dataset - the dev set. If cross_val is True, each dataset
is a list of DataSet objects; Otherwise, each dataset is a DataSet object.
"""
-
if pickle_exist(pickle_path, "word2id.pkl") and pickle_exist(pickle_path, "class2id.pkl"):
self.data_vocab = load_pickle(pickle_path, "word2id.pkl")
self.label_vocab = load_pickle(pickle_path, "class2id.pkl")
@@ -176,6 +197,16 @@ class Preprocessor(object):
self.label_vocab.update(label)
return self.data_vocab, self.label_vocab
+ def build_char_dict(self):
+ char_collection = set()
+ for word in self.data_vocab.word2idx:
+ if len(word) == 0:
+ continue
+ for ch in word:
+ if ch not in char_collection:
+ char_collection.add(ch)
+ self.character_vocab.update(list(char_collection))
+
def build_reverse_dict(self):
self.data_vocab.build_reverse_vocab()
self.label_vocab.build_reverse_vocab()
@@ -277,11 +308,3 @@ class ClassPreprocess(Preprocessor):
print("[FastNLP warning] ClassPreprocess is about to deprecate. Please use Preprocess directly.")
super(ClassPreprocess, self).__init__()
-
-if __name__ == "__main__":
- p = Preprocessor()
- train_dev_data = [[["I", "am", "a", "good", "student", "."], "0"],
- [["You", "are", "pretty", "."], "1"]
- ]
- training_set = p.run(train_dev_data)
- print(training_set)
diff --git a/fastNLP/core/tester.py b/fastNLP/core/tester.py
index 0a75f46a..0e74145b 100644
--- a/fastNLP/core/tester.py
+++ b/fastNLP/core/tester.py
@@ -1,7 +1,7 @@
-import numpy as np
import torch
from fastNLP.core.batch import Batch
+from fastNLP.core.metrics import Evaluator
from fastNLP.core.sampler import RandomSampler
from fastNLP.saver.logger import create_logger
@@ -22,28 +22,23 @@ class Tester(object):
"kwargs" must have the same type as "default_args" on corresponding keys.
Otherwise, error will raise.
"""
- default_args = {"save_output": True, # collect outputs of validation set
- "save_loss": True, # collect losses in validation
- "save_best_dev": False, # save best model during validation
- "batch_size": 8,
+ default_args = {"batch_size": 8,
"use_cuda": False,
"pickle_path": "./save/",
"model_name": "dev_best_model.pkl",
- "print_every_step": 1,
+ "evaluator": Evaluator()
}
"""
"required_args" is the collection of arguments that users must pass to Trainer explicitly.
This is used to warn users of essential settings in the training.
Specially, "required_args" does not have default value, so they have nothing to do with "default_args".
"""
- required_args = {"task" # one of ("seq_label", "text_classify")
- }
+ required_args = {}
for req_key in required_args:
if req_key not in kwargs:
logger.error("Tester lacks argument {}".format(req_key))
raise ValueError("Tester lacks argument {}".format(req_key))
- self._task = kwargs["task"]
for key in default_args:
if key in kwargs:
@@ -59,17 +54,13 @@ class Tester(object):
pass
print(default_args)
- self.save_output = default_args["save_output"]
- self.save_best_dev = default_args["save_best_dev"]
- self.save_loss = default_args["save_loss"]
self.batch_size = default_args["batch_size"]
self.pickle_path = default_args["pickle_path"]
self.use_cuda = default_args["use_cuda"]
- self.print_every_step = default_args["print_every_step"]
+ self._evaluator = default_args["evaluator"]
self._model = None
self.eval_history = [] # evaluation results of all batches
- self.batch_output = [] # outputs of all batches
def test(self, network, dev_data):
if torch.cuda.is_available() and self.use_cuda:
@@ -80,26 +71,18 @@ class Tester(object):
# turn on the testing mode; clean up the history
self.mode(network, is_test=True)
self.eval_history.clear()
- self.batch_output.clear()
+ output_list = []
+ truth_list = []
data_iterator = Batch(dev_data, self.batch_size, sampler=RandomSampler(), use_cuda=self.use_cuda)
- step = 0
for batch_x, batch_y in data_iterator:
with torch.no_grad():
prediction = self.data_forward(network, batch_x)
- eval_results = self.evaluate(prediction, batch_y)
-
- if self.save_output:
- self.batch_output.append(prediction)
- if self.save_loss:
- self.eval_history.append(eval_results)
-
- print_output = "[test step {}] {}".format(step, eval_results)
- logger.info(print_output)
- if self.print_every_step > 0 and step % self.print_every_step == 0:
- print(self.make_eval_output(prediction, eval_results))
- step += 1
+ output_list.append(prediction)
+ truth_list.append(batch_y)
+ eval_results = self.evaluate(output_list, truth_list)
+ print("[tester] {}".format(self.print_eval_results(eval_results)))
def mode(self, model, is_test=False):
"""Train mode or Test mode. This is for PyTorch currently.
@@ -121,104 +104,30 @@ class Tester(object):
def evaluate(self, predict, truth):
"""Compute evaluation metrics.
- :param predict: Tensor
- :param truth: Tensor
+ :param predict: list of Tensor
+ :param truth: list of dict
:return eval_results: can be anything. It will be stored in self.eval_history
"""
- if "label_seq" in truth:
- truth = truth["label_seq"]
- elif "label" in truth:
- truth = truth["label"]
- else:
- raise NotImplementedError("Unknown key {} in batch_y.".format(truth.keys()))
+ return self._evaluator(predict, truth)
- if self._task == "seq_label":
- return self._seq_label_evaluate(predict, truth)
- elif self._task == "text_classify":
- return self._text_classify_evaluate(predict, truth)
- else:
- raise NotImplementedError("Unknown task type {}.".format(self._task))
-
- def _seq_label_evaluate(self, predict, truth):
- batch_size, max_len = predict.size(0), predict.size(1)
- loss = self._model.loss(predict, truth) / batch_size
- prediction = self._model.prediction(predict)
- # pad prediction to equal length
- for pred in prediction:
- if len(pred) < max_len:
- pred += [0] * (max_len - len(pred))
- results = torch.Tensor(prediction).view(-1, )
-
- # make sure "results" is in the same device as "truth"
- results = results.to(truth)
- accuracy = torch.sum(results == truth.view((-1,))).to(torch.float) / results.shape[0]
- return [float(loss), float(accuracy)]
-
- def _text_classify_evaluate(self, y_logit, y_true):
- y_prob = torch.nn.functional.softmax(y_logit, dim=-1)
- return [y_prob, y_true]
-
- @property
- def metrics(self):
- """Compute and return metrics.
- Use self.eval_history to compute metrics over the whole dev set.
- Please refer to metrics.py for common metric functions.
-
- :return : variable number of outputs
- """
- if self._task == "seq_label":
- return self._seq_label_metrics
- elif self._task == "text_classify":
- return self._text_classify_metrics
- else:
- raise NotImplementedError("Unknown task type {}.".format(self._task))
-
- @property
- def _seq_label_metrics(self):
- batch_loss = np.mean([x[0] for x in self.eval_history])
- batch_accuracy = np.mean([x[1] for x in self.eval_history])
- return batch_loss, batch_accuracy
-
- @property
- def _text_classify_metrics(self):
- y_prob, y_true = zip(*self.eval_history)
- y_prob = torch.cat(y_prob, dim=0)
- y_pred = torch.argmax(y_prob, dim=-1)
- y_true = torch.cat(y_true, dim=0)
- acc = float(torch.sum(y_pred == y_true)) / len(y_true)
- return y_true.cpu().numpy(), y_prob.cpu().numpy(), acc
-
- def show_metrics(self):
- """Customize evaluation outputs in Trainer.
- Called by Trainer to print evaluation results on dev set during training.
- Use self.metrics to fetch available metrics.
-
- :return print_str: str
- """
- loss, accuracy = self.metrics
- return "dev loss={:.2f}, accuracy={:.2f}".format(loss, accuracy)
+ def print_eval_results(self, results):
+ """Override this method to support more print formats.
- def make_eval_output(self, predictions, eval_results):
- """Customize Tester outputs.
+ :param results: dict, (str: float) is (metrics name: value)
- :param predictions: Tensor
- :param eval_results: Tensor
- :return: str, to be printed.
"""
- return self.show_metrics()
+ return ", ".join([str(key) + "=" + str(value) for key, value in results.items()])
class SeqLabelTester(Tester):
def __init__(self, **test_args):
- test_args.update({"task": "seq_label"})
print(
- "[FastNLP Warning] SeqLabelTester will be deprecated. Please use Tester with argument 'task'='seq_label'.")
+ "[FastNLP Warning] SeqLabelTester will be deprecated. Please use Tester directly.")
super(SeqLabelTester, self).__init__(**test_args)
class ClassificationTester(Tester):
def __init__(self, **test_args):
- test_args.update({"task": "text_classify"})
print(
- "[FastNLP Warning] ClassificationTester will be deprecated. Please use Tester with argument 'task'='text_classify'.")
+ "[FastNLP Warning] ClassificationTester will be deprecated. Please use Tester directly.")
super(ClassificationTester, self).__init__(**test_args)
diff --git a/fastNLP/core/trainer.py b/fastNLP/core/trainer.py
index a73229b2..957a4757 100644
--- a/fastNLP/core/trainer.py
+++ b/fastNLP/core/trainer.py
@@ -1,4 +1,3 @@
-import copy
import os
import time
from datetime import timedelta
@@ -8,6 +7,7 @@ from tensorboardX import SummaryWriter
from fastNLP.core.batch import Batch
from fastNLP.core.loss import Loss
+from fastNLP.core.metrics import Evaluator
from fastNLP.core.optimizer import Optimizer
from fastNLP.core.sampler import RandomSampler
from fastNLP.core.tester import SeqLabelTester, ClassificationTester
@@ -43,21 +43,20 @@ class Trainer(object):
default_args = {"epochs": 1, "batch_size": 2, "validate": False, "use_cuda": False, "pickle_path": "./save/",
"save_best_dev": False, "model_name": "default_model_name.pkl", "print_every_step": 1,
"loss": Loss(None), # used to pass type check
- "optimizer": Optimizer("Adam", lr=0.001, weight_decay=0)
+ "optimizer": Optimizer("Adam", lr=0.001, weight_decay=0),
+ "evaluator": Evaluator()
}
"""
"required_args" is the collection of arguments that users must pass to Trainer explicitly.
This is used to warn users of essential settings in the training.
Specially, "required_args" does not have default value, so they have nothing to do with "default_args".
"""
- required_args = {"task" # one of ("seq_label", "text_classify")
- }
+ required_args = {}
for req_key in required_args:
if req_key not in kwargs:
logger.error("Trainer lacks argument {}".format(req_key))
raise ValueError("Trainer lacks argument {}".format(req_key))
- self._task = kwargs["task"]
for key in default_args:
if key in kwargs:
@@ -86,6 +85,7 @@ class Trainer(object):
self._loss_func = default_args["loss"].get() # return a pytorch loss function or None
self._optimizer = None
self._optimizer_proto = default_args["optimizer"]
+ self._evaluator = default_args["evaluator"]
self._summary_writer = SummaryWriter(self.pickle_path + 'tensorboard_logs')
self._graph_summaried = False
self._best_accuracy = 0.0
@@ -106,9 +106,8 @@ class Trainer(object):
# define Tester over dev data
if self.validate:
- default_valid_args = {"save_output": True, "validate_in_training": True, "save_dev_input": True,
- "save_loss": True, "batch_size": self.batch_size, "pickle_path": self.pickle_path,
- "use_cuda": self.use_cuda, "print_every_step": 0}
+ default_valid_args = {"batch_size": self.batch_size, "pickle_path": self.pickle_path,
+ "use_cuda": self.use_cuda, "evaluator": self._evaluator}
validator = self._create_validator(default_valid_args)
logger.info("validator defined as {}".format(str(validator)))
@@ -142,15 +141,6 @@ class Trainer(object):
logger.info("validation started")
validator.test(network, dev_data)
- if self.save_best_dev and self.best_eval_result(validator):
- self.save_model(network, self.model_name)
- print("Saved better model selected by validation.")
- logger.info("Saved better model selected by validation.")
-
- valid_results = validator.show_metrics()
- print("[epoch {}] {}".format(epoch, valid_results))
- logger.info("[epoch {}] {}".format(epoch, valid_results))
-
def _train_step(self, data_iterator, network, **kwargs):
"""Training process in one epoch.
@@ -178,31 +168,6 @@ class Trainer(object):
logger.info(print_output)
step += 1
- def cross_validate(self, network, train_data_cv, dev_data_cv):
- """Training with cross validation.
-
- :param network: the model
- :param train_data_cv: four-level list, of shape [num_folds, num_examples, 2, ?]
- :param dev_data_cv: four-level list, of shape [num_folds, num_examples, 2, ?]
-
- """
- if len(train_data_cv) != len(dev_data_cv):
- logger.error("the number of folds in train and dev data unequals {}!={}".format(len(train_data_cv),
- len(dev_data_cv)))
- raise RuntimeError("the number of folds in train and dev data unequals")
- if self.validate is False:
- logger.warn("Cross validation requires self.validate to be True. Please turn it on. ")
- print("[warning] Cross validation requires self.validate to be True. Please turn it on. ")
- self.validate = True
-
- n_fold = len(train_data_cv)
- logger.info("perform {} folds cross validation.".format(n_fold))
- for i in range(n_fold):
- print("CV:", i)
- logger.info("running the {} of {} folds cross validation".format(i + 1, n_fold))
- network_copy = copy.deepcopy(network)
- self.train(network_copy, train_data_cv[i], dev_data_cv[i])
-
def mode(self, model, is_test=False):
"""Train mode or Test mode. This is for PyTorch currently.
@@ -229,18 +194,9 @@ class Trainer(object):
self._optimizer.step()
def data_forward(self, network, x):
- if self._task == "seq_label":
- y = network(x["word_seq"], x["word_seq_origin_len"])
- elif self._task == "text_classify":
- y = network(x["word_seq"])
- else:
- raise NotImplementedError("Unknown task type {}.".format(self._task))
-
+ y = network(**x)
if not self._graph_summaried:
- if self._task == "seq_label":
- self._summary_writer.add_graph(network, (x["word_seq"], x["word_seq_origin_len"]), verbose=False)
- elif self._task == "text_classify":
- self._summary_writer.add_graph(network, x["word_seq"], verbose=False)
+ # self._summary_writer.add_graph(network, x, verbose=False)
self._graph_summaried = True
return y
@@ -261,13 +217,9 @@ class Trainer(object):
:param truth: ground truth label vector
:return: a scalar
"""
- if "label_seq" in truth:
- truth = truth["label_seq"]
- elif "label" in truth:
- truth = truth["label"]
- truth = truth.view((-1,))
- else:
- raise NotImplementedError("Unknown key {} in batch_y.".format(truth.keys()))
+ if len(truth) > 1:
+ raise NotImplementedError("Not ready to handle multi-labels.")
+ truth = list(truth.values())[0] if len(truth) > 0 else None
return self._loss_func(predict, truth)
def define_loss(self):
@@ -278,8 +230,8 @@ class Trainer(object):
These two losses cannot be defined at the same time.
Trainer does not handle loss definition or choose default losses.
"""
- if hasattr(self._model, "loss") and self._loss_func is not None:
- raise ValueError("Both the model and Trainer define loss. Please take out your loss.")
+ # if hasattr(self._model, "loss") and self._loss_func is not None:
+ # raise ValueError("Both the model and Trainer define loss. Please take out your loss.")
if hasattr(self._model, "loss"):
self._loss_func = self._model.loss
@@ -322,9 +274,8 @@ class SeqLabelTrainer(Trainer):
"""
def __init__(self, **kwargs):
- kwargs.update({"task": "seq_label"})
print(
- "[FastNLP Warning] SeqLabelTrainer will be deprecated. Please use Trainer with argument 'task'='seq_label'.")
+ "[FastNLP Warning] SeqLabelTrainer will be deprecated. Please use Trainer directly.")
super(SeqLabelTrainer, self).__init__(**kwargs)
def _create_validator(self, valid_args):
@@ -335,9 +286,8 @@ class ClassificationTrainer(Trainer):
"""Trainer for text classification."""
def __init__(self, **train_args):
- train_args.update({"task": "text_classify"})
print(
- "[FastNLP Warning] ClassificationTrainer will be deprecated. Please use Trainer with argument 'task'='text_classify'.")
+ "[FastNLP Warning] ClassificationTrainer will be deprecated. Please use Trainer directly.")
super(ClassificationTrainer, self).__init__(**train_args)
def _create_validator(self, valid_args):
diff --git a/fastNLP/core/vocabulary.py b/fastNLP/core/vocabulary.py
index ad618ff9..08c00644 100644
--- a/fastNLP/core/vocabulary.py
+++ b/fastNLP/core/vocabulary.py
@@ -10,13 +10,15 @@ DEFAULT_WORD_TO_INDEX = {DEFAULT_PADDING_LABEL: 0, DEFAULT_UNKNOWN_LABEL: 1,
DEFAULT_RESERVED_LABEL[0]: 2, DEFAULT_RESERVED_LABEL[1]: 3,
DEFAULT_RESERVED_LABEL[2]: 4}
+
def isiterable(p_object):
try:
it = iter(p_object)
- except TypeError:
+ except TypeError:
return False
return True
+
class Vocabulary(object):
"""Use for word and index one to one mapping
@@ -28,9 +30,11 @@ class Vocabulary(object):
vocab["word"]
vocab.to_word(5)
"""
+
def __init__(self, need_default=True):
"""
- :param bool need_default: set if the Vocabulary has default labels reserved.
+ :param bool need_default: set if the Vocabulary has default labels reserved for sequences. Default: True.
+
"""
if need_default:
self.word2idx = deepcopy(DEFAULT_WORD_TO_INDEX)
@@ -50,20 +54,19 @@ class Vocabulary(object):
def update(self, word):
"""add word or list of words into Vocabulary
- :param word: a list of str or str
+ :param word: a list of string or a single string
"""
if not isinstance(word, str) and isiterable(word):
- # it's a nested list
+ # it's a nested list
for w in word:
self.update(w)
else:
- # it's a word to be added
+ # it's a word to be added
if word not in self.word2idx:
self.word2idx[word] = len(self)
if self.idx2word is not None:
self.idx2word = None
-
def __getitem__(self, w):
"""To support usage like::
@@ -81,12 +84,12 @@ class Vocabulary(object):
:param str w:
"""
return self[w]
-
+
def unknown_idx(self):
- if self.unknown_label is None:
+ if self.unknown_label is None:
return None
return self.word2idx[self.unknown_label]
-
+
def padding_idx(self):
if self.padding_label is None:
return None
@@ -95,8 +98,8 @@ class Vocabulary(object):
def build_reverse_vocab(self):
"""build 'index to word' dict based on 'word to index' dict
"""
- self.idx2word = {self.word2idx[w] : w for w in self.word2idx}
-
+ self.idx2word = {self.word2idx[w]: w for w in self.word2idx}
+
def to_word(self, idx):
"""given a word's index, return the word itself
@@ -105,7 +108,7 @@ class Vocabulary(object):
if self.idx2word is None:
self.build_reverse_vocab()
return self.idx2word[idx]
-
+
def __getstate__(self):
"""use to prepare data for pickle
"""
@@ -113,12 +116,9 @@ class Vocabulary(object):
# no need to pickle idx2word as it can be constructed from word2idx
del state['idx2word']
return state
-
+
def __setstate__(self, state):
"""use to restore state from pickle
"""
self.__dict__.update(state)
self.idx2word = None
-
-
-
\ No newline at end of file
diff --git a/fastNLP/fastnlp.py b/fastNLP/fastnlp.py
index 4643c247..0bd56d18 100644
--- a/fastNLP/fastnlp.py
+++ b/fastNLP/fastnlp.py
@@ -1,5 +1,6 @@
import os
+from fastNLP.core.dataset import SeqLabelDataSet, TextClassifyDataSet
from fastNLP.core.predictor import SeqLabelInfer, ClassificationInfer
from fastNLP.core.preprocess import load_pickle
from fastNLP.loader.config_loader import ConfigLoader, ConfigSection
@@ -71,11 +72,13 @@ class FastNLP(object):
:param model_dir: this directory should contain the following files:
1. a trained model
2. a config file, which is a fastNLP's configuration.
- 3. a Vocab file, which is a pickle object of a Vocab instance.
+ 3. two Vocab files, which are pickle objects of Vocab instances, representing feature and label vocabs.
"""
self.model_dir = model_dir
self.model = None
self.infer_type = None # "seq_label"/"text_class"
+ self.word_vocab = None
+ self.label_vocab = None
def load(self, model_name, config_file="config", section_name="model"):
"""
@@ -100,10 +103,10 @@ class FastNLP(object):
print("Restore model hyper-parameters {}".format(str(model_args.data)))
# fetch dictionary size and number of labels from pickle files
- word_vocab = load_pickle(self.model_dir, "word2id.pkl")
- model_args["vocab_size"] = len(word_vocab)
- label_vocab = load_pickle(self.model_dir, "class2id.pkl")
- model_args["num_classes"] = len(label_vocab)
+ self.word_vocab = load_pickle(self.model_dir, "word2id.pkl")
+ model_args["vocab_size"] = len(self.word_vocab)
+ self.label_vocab = load_pickle(self.model_dir, "label2id.pkl")
+ model_args["num_classes"] = len(self.label_vocab)
# Construct the model
model = model_class(model_args)
@@ -130,8 +133,11 @@ class FastNLP(object):
# tokenize: list of string ---> 2-D list of string
infer_input = self.tokenize(raw_input, language="zh")
- # 2-D list of string ---> 2-D list of tags
- results = infer.predict(self.model, infer_input)
+ # create DataSet: 2-D list of strings ----> DataSet
+ infer_data = self._create_data_set(infer_input)
+
+ # DataSet ---> 2-D list of tags
+ results = infer.predict(self.model, infer_data)
# 2-D list of tags ---> list of final answers
outputs = self._make_output(results, infer_input)
@@ -154,6 +160,11 @@ class FastNLP(object):
return module
def _create_inference(self, model_dir):
+ """Specify which task to perform.
+
+ :param model_dir:
+ :return:
+ """
if self.infer_type == "seq_label":
return SeqLabelInfer(model_dir)
elif self.infer_type == "text_class":
@@ -161,8 +172,26 @@ class FastNLP(object):
else:
raise ValueError("fail to create inference instance")
+ def _create_data_set(self, infer_input):
+ """Create a DataSet object given the raw inputs.
+
+ :param infer_input: 2-D lists of strings
+ :return data_set: a DataSet object
+ """
+ if self.infer_type == "seq_label":
+ data_set = SeqLabelDataSet()
+ data_set.load_raw(infer_input, {"word_vocab": self.word_vocab})
+ return data_set
+ elif self.infer_type == "text_class":
+ data_set = TextClassifyDataSet()
+ data_set.load_raw(infer_input, {"word_vocab": self.word_vocab})
+ return data_set
+ else:
+ raise RuntimeError("fail to make outputs with infer type {}".format(self.infer_type))
+
+
def _load(self, model_dir, model_name):
- # To do
+
return 0
def _download(self, model_name, url):
@@ -172,7 +201,7 @@ class FastNLP(object):
:param url:
"""
print("Downloading {} from {}".format(model_name, url))
- # To do
+ # TODO: download model via url
def model_exist(self, model_dir):
"""
diff --git a/fastNLP/loader/base_loader.py b/fastNLP/loader/base_loader.py
index 808567fb..fc2814c8 100644
--- a/fastNLP/loader/base_loader.py
+++ b/fastNLP/loader/base_loader.py
@@ -1,27 +1,24 @@
class BaseLoader(object):
- """docstring for BaseLoader"""
- def __init__(self, data_path):
+ def __init__(self):
super(BaseLoader, self).__init__()
- self.data_path = data_path
-
- def load(self):
- """
- :return: string
- """
- with open(self.data_path, "r", encoding="utf-8") as f:
- text = f.read()
- return text
- def load_lines(self):
- with open(self.data_path, "r", encoding="utf=8") as f:
+ @staticmethod
+ def load_lines(data_path):
+ with open(data_path, "r", encoding="utf=8") as f:
text = f.readlines()
return [line.strip() for line in text]
+ @staticmethod
+ def load(data_path):
+ with open(data_path, "r", encoding="utf-8") as f:
+ text = f.readlines()
+ return [[word for word in sent.strip()] for sent in text]
+
class ToyLoader0(BaseLoader):
"""
- For charLM
+ For CharLM
"""
def __init__(self, data_path):
diff --git a/fastNLP/loader/config_loader.py b/fastNLP/loader/config_loader.py
index 94871222..9818d411 100644
--- a/fastNLP/loader/config_loader.py
+++ b/fastNLP/loader/config_loader.py
@@ -8,9 +8,9 @@ from fastNLP.loader.base_loader import BaseLoader
class ConfigLoader(BaseLoader):
"""loader for configuration files"""
- def __int__(self, data_name, data_path):
- super(ConfigLoader, self).__init__(data_path)
- self.config = self.parse(super(ConfigLoader, self).load())
+ def __int__(self, data_path):
+ super(ConfigLoader, self).__init__()
+ self.config = self.parse(super(ConfigLoader, self).load(data_path))
@staticmethod
def parse(string):
diff --git a/fastNLP/loader/dataset_loader.py b/fastNLP/loader/dataset_loader.py
index 72da209c..a6a0fb77 100644
--- a/fastNLP/loader/dataset_loader.py
+++ b/fastNLP/loader/dataset_loader.py
@@ -3,14 +3,17 @@ import os
from fastNLP.loader.base_loader import BaseLoader
-class DatasetLoader(BaseLoader):
+class DataSetLoader(BaseLoader):
""""loader for data sets"""
- def __init__(self, data_path):
- super(DatasetLoader, self).__init__(data_path)
+ def __init__(self):
+ super(DataSetLoader, self).__init__()
+ def load(self, path):
+ raise NotImplementedError
-class POSDatasetLoader(DatasetLoader):
+
+class POSDataSetLoader(DataSetLoader):
"""Dataset Loader for POS Tag datasets.
In these datasets, each line are divided by '\t'
@@ -31,16 +34,10 @@ class POSDatasetLoader(DatasetLoader):
to label5.
"""
- def __init__(self, data_path):
- super(POSDatasetLoader, self).__init__(data_path)
-
- def load(self):
- assert os.path.exists(self.data_path)
- with open(self.data_path, "r", encoding="utf-8") as f:
- line = f.read()
- return line
+ def __init__(self):
+ super(POSDataSetLoader, self).__init__()
- def load_lines(self):
+ def load(self, data_path):
"""
:return data: three-level list
[
@@ -49,7 +46,7 @@ class POSDatasetLoader(DatasetLoader):
...
]
"""
- with open(self.data_path, "r", encoding="utf-8") as f:
+ with open(data_path, "r", encoding="utf-8") as f:
lines = f.readlines()
return self.parse(lines)
@@ -79,15 +76,16 @@ class POSDatasetLoader(DatasetLoader):
return data
-class TokenizeDatasetLoader(DatasetLoader):
+class TokenizeDataSetLoader(DataSetLoader):
"""
Data set loader for tokenization data sets
"""
- def __init__(self, data_path):
- super(TokenizeDatasetLoader, self).__init__(data_path)
+ def __init__(self):
+ super(TokenizeDataSetLoader, self).__init__()
- def load_pku(self, max_seq_len=32):
+ @staticmethod
+ def load(data_path, max_seq_len=32):
"""
load pku dataset for Chinese word segmentation
CWS (Chinese Word Segmentation) pku training dataset format:
@@ -104,7 +102,7 @@ class TokenizeDatasetLoader(DatasetLoader):
:return: three-level lists
"""
assert isinstance(max_seq_len, int) and max_seq_len > 0
- with open(self.data_path, "r", encoding="utf-8") as f:
+ with open(data_path, "r", encoding="utf-8") as f:
sentences = f.readlines()
data = []
for sent in sentences:
@@ -135,15 +133,15 @@ class TokenizeDatasetLoader(DatasetLoader):
return data
-class ClassDatasetLoader(DatasetLoader):
+class ClassDataSetLoader(DataSetLoader):
"""Loader for classification data sets"""
- def __init__(self, data_path):
- super(ClassDatasetLoader, self).__init__(data_path)
+ def __init__(self):
+ super(ClassDataSetLoader, self).__init__()
- def load(self):
- assert os.path.exists(self.data_path)
- with open(self.data_path, "r", encoding="utf-8") as f:
+ def load(self, data_path):
+ assert os.path.exists(data_path)
+ with open(data_path, "r", encoding="utf-8") as f:
lines = f.readlines()
return self.parse(lines)
@@ -169,21 +167,21 @@ class ClassDatasetLoader(DatasetLoader):
return dataset
-class ConllLoader(DatasetLoader):
+class ConllLoader(DataSetLoader):
"""loader for conll format files"""
def __int__(self, data_path):
"""
:param str data_path: the path to the conll data set
"""
- super(ConllLoader, self).__init__(data_path)
- self.data_set = self.parse(self.load())
+ super(ConllLoader, self).__init__()
+ self.data_set = self.parse(self.load(data_path))
- def load(self):
+ def load(self, data_path):
"""
:return: list lines: all lines in a conll file
"""
- with open(self.data_path, "r", encoding="utf-8") as f:
+ with open(data_path, "r", encoding="utf-8") as f:
lines = f.readlines()
return lines
@@ -207,28 +205,48 @@ class ConllLoader(DatasetLoader):
return sentences
-class LMDatasetLoader(DatasetLoader):
- def __init__(self, data_path):
- super(LMDatasetLoader, self).__init__(data_path)
+class LMDataSetLoader(DataSetLoader):
+ """Language Model Dataset Loader
- def load(self):
- if not os.path.exists(self.data_path):
- raise FileNotFoundError("file {} not found.".format(self.data_path))
- with open(self.data_path, "r", encoding="utf=8") as f:
- text = " ".join(f.readlines())
- return text.strip().split()
+ This loader produces data for language model training in a supervised way.
+ That means it has X and Y.
+ """
+
+ def __init__(self):
+ super(LMDataSetLoader, self).__init__()
-class PeopleDailyCorpusLoader(DatasetLoader):
+ def load(self, data_path):
+ if not os.path.exists(data_path):
+ raise FileNotFoundError("file {} not found.".format(data_path))
+ with open(data_path, "r", encoding="utf=8") as f:
+ text = " ".join(f.readlines())
+ tokens = text.strip().split()
+ return self.sentence_cut(tokens)
+
+ def sentence_cut(self, tokens, sentence_length=15):
+ start_idx = 0
+ data_set = []
+ for idx in range(len(tokens) // sentence_length):
+ x = tokens[start_idx * idx: start_idx * idx + sentence_length]
+ y = tokens[start_idx * idx + 1: start_idx * idx + sentence_length + 1]
+ if start_idx * idx + sentence_length + 1 >= len(tokens):
+ # ad hoc
+ y.extend(["