From 9d43239fc17a8ec6029b5ef20f175cd4a6d9008b Mon Sep 17 00:00:00 2001 From: xuyige Date: Sun, 21 Apr 2019 15:41:20 +0800 Subject: [PATCH 1/9] update attention --- fastNLP/models/snli.py | 38 ++++++++++++------------- fastNLP/modules/aggregator/__init__.py | 2 +- fastNLP/modules/aggregator/attention.py | 36 ++++++++++++++++------- fastNLP/modules/encoder/transformer.py | 4 +-- 4 files changed, 47 insertions(+), 33 deletions(-) diff --git a/fastNLP/models/snli.py b/fastNLP/models/snli.py index 6a7d8d84..5816d2af 100644 --- a/fastNLP/models/snli.py +++ b/fastNLP/models/snli.py @@ -1,6 +1,5 @@ import torch import torch.nn as nn -import torch.nn.functional as F from fastNLP.models.base_model import BaseModel from fastNLP.modules import decoder as Decoder @@ -40,7 +39,7 @@ class ESIM(BaseModel): batch_first=self.batch_first, bidirectional=True ) - self.bi_attention = Aggregator.Bi_Attention() + self.bi_attention = Aggregator.BiAttention() self.mean_pooling = Aggregator.MeanPoolWithMask() self.max_pooling = Aggregator.MaxPoolWithMask() @@ -53,23 +52,23 @@ class ESIM(BaseModel): self.output = Decoder.MLP([4 * self.hidden_size, self.hidden_size, self.n_labels], 'tanh', dropout=self.dropout) - def forward(self, premise, hypothesis, premise_len, hypothesis_len): + def forward(self, words1, words2, seq_len1, seq_len2): """ Forward function - :param premise: A Tensor represents premise: [batch size(B), premise seq len(PL)]. - :param hypothesis: A Tensor represents hypothesis: [B, hypothesis seq len(HL)]. - :param premise_len: A Tensor record which is a real word and which is a padding word in premise: [B, PL]. - :param hypothesis_len: A Tensor record which is a real word and which is a padding word in hypothesis: [B, HL]. + :param words1: A Tensor represents premise: [batch size(B), premise seq len(PL)]. + :param words2: A Tensor represents hypothesis: [B, hypothesis seq len(HL)]. + :param seq_len1: A Tensor record which is a real word and which is a padding word in premise: [B]. + :param seq_len2: A Tensor record which is a real word and which is a padding word in hypothesis: [B]. :return: prediction: A Dict with Tensor of classification result: [B, n_labels(N)]. """ - premise0 = self.embedding_layer(self.embedding(premise)) - hypothesis0 = self.embedding_layer(self.embedding(hypothesis)) + premise0 = self.embedding_layer(self.embedding(words1)) + hypothesis0 = self.embedding_layer(self.embedding(words2)) _BP, _PSL, _HP = premise0.size() _BH, _HSL, _HH = hypothesis0.size() - _BPL, _PLL = premise_len.size() - _HPL, _HLL = hypothesis_len.size() + _BPL, _PLL = seq_len1.size() + _HPL, _HLL = seq_len2.size() assert _BP == _BH and _BPL == _HPL and _BP == _BPL assert _HP == _HH @@ -84,7 +83,7 @@ class ESIM(BaseModel): a = torch.mean(a0.view(B, PL, -1, H), dim=2) # a: [B, PL, H] b = torch.mean(b0.view(B, HL, -1, H), dim=2) # b: [B, HL, H] - ai, bi = self.bi_attention(a, b, premise_len, hypothesis_len) + ai, bi = self.bi_attention(a, b, seq_len1, seq_len2) ma = torch.cat((a, ai, a - ai, a * ai), dim=2) # ma: [B, PL, 4 * H] mb = torch.cat((b, bi, b - bi, b * bi), dim=2) # mb: [B, HL, 4 * H] @@ -98,17 +97,18 @@ class ESIM(BaseModel): va = torch.mean(vat.view(B, PL, -1, H), dim=2) # va: [B, PL, H] vb = torch.mean(vbt.view(B, HL, -1, H), dim=2) # vb: [B, HL, H] - va_ave = self.mean_pooling(va, premise_len, dim=1) # va_ave: [B, H] - va_max, va_arg_max = self.max_pooling(va, premise_len, dim=1) # va_max: [B, H] - vb_ave = self.mean_pooling(vb, hypothesis_len, dim=1) # vb_ave: [B, H] - vb_max, vb_arg_max = self.max_pooling(vb, hypothesis_len, dim=1) # vb_max: [B, H] + va_ave = self.mean_pooling(va, seq_len1, dim=1) # va_ave: [B, H] + va_max, va_arg_max = self.max_pooling(va, seq_len1, dim=1) # va_max: [B, H] + vb_ave = self.mean_pooling(vb, seq_len2, dim=1) # vb_ave: [B, H] + vb_max, vb_arg_max = self.max_pooling(vb, seq_len2, dim=1) # vb_max: [B, H] v = torch.cat((va_ave, va_max, vb_ave, vb_max), dim=1) # v: [B, 4 * H] - prediction = F.tanh(self.output(v)) # prediction: [B, N] + prediction = torch.tanh(self.output(v)) # prediction: [B, N] return {'pred': prediction} - def predict(self, premise, hypothesis, premise_len, hypothesis_len): - return self.forward(premise, hypothesis, premise_len, hypothesis_len) + def predict(self, words1, words2, seq_len1, seq_len2): + prediction = self.forward(words1, words2, seq_len1, seq_len2)['pred'] + return torch.argmax(prediction, dim=-1) diff --git a/fastNLP/modules/aggregator/__init__.py b/fastNLP/modules/aggregator/__init__.py index 2fabb89e..43d60cac 100644 --- a/fastNLP/modules/aggregator/__init__.py +++ b/fastNLP/modules/aggregator/__init__.py @@ -5,6 +5,6 @@ from .avg_pool import MeanPoolWithMask from .kmax_pool import KMaxPool from .attention import Attention -from .attention import Bi_Attention +from .attention import BiAttention from .self_attention import SelfAttention diff --git a/fastNLP/modules/aggregator/attention.py b/fastNLP/modules/aggregator/attention.py index ef9d159d..33d73a07 100644 --- a/fastNLP/modules/aggregator/attention.py +++ b/fastNLP/modules/aggregator/attention.py @@ -23,9 +23,9 @@ class Attention(torch.nn.Module): raise NotImplementedError -class DotAtte(nn.Module): +class DotAttention(nn.Module): def __init__(self, key_size, value_size, dropout=0.1): - super(DotAtte, self).__init__() + super(DotAttention, self).__init__() self.key_size = key_size self.value_size = value_size self.scale = math.sqrt(key_size) @@ -48,7 +48,7 @@ class DotAtte(nn.Module): return torch.matmul(output, V) -class MultiHeadAtte(nn.Module): +class MultiHeadAttention(nn.Module): def __init__(self, input_size, key_size, value_size, num_head, dropout=0.1): """ @@ -58,7 +58,7 @@ class MultiHeadAtte(nn.Module): :param num_head: int，head的数量。 :param dropout: float。 """ - super(MultiHeadAtte, self).__init__() + super(MultiHeadAttention, self).__init__() self.input_size = input_size self.key_size = key_size self.value_size = value_size @@ -68,7 +68,7 @@ class MultiHeadAtte(nn.Module): self.q_in = nn.Linear(input_size, in_size) self.k_in = nn.Linear(input_size, in_size) self.v_in = nn.Linear(input_size, in_size) - self.attention = DotAtte(key_size=key_size, value_size=value_size) + self.attention = DotAttention(key_size=key_size, value_size=value_size) self.out = nn.Linear(value_size * num_head, input_size) self.drop = TimestepDropout(dropout) self.reset_parameters() @@ -109,16 +109,30 @@ class MultiHeadAtte(nn.Module): return output -class Bi_Attention(nn.Module): +class BiAttention(nn.Module): + """Bi Attention module + Calculate Bi Attention matrix `e` + .. math:: + \begin{array}{ll} \\ + e_ij = {a}^{\mathbf{T}}_{i}{b}_{j} \\ + a_i = + b_j = + \end{array} + """ + def __init__(self): - super(Bi_Attention, self).__init__() + super(BiAttention, self).__init__() self.inf = 10e12 def forward(self, in_x1, in_x2, x1_len, x2_len): - # in_x1: [batch_size, x1_seq_len, hidden_size] - # in_x2: [batch_size, x2_seq_len, hidden_size] - # x1_len: [batch_size, x1_seq_len] - # x2_len: [batch_size, x2_seq_len] + """ + :param torch.Tensor in_x1: [batch_size, x1_seq_len, hidden_size] 第一句的特征表示 + :param torch.Tensor in_x2: [batch_size, x2_seq_len, hidden_size] 第二句的特征表示 + :param torch.Tensor x1_len: [batch_size, x1_seq_len] 第一句的0/1mask矩阵 + :param torch.Tensor x2_len: [batch_size, x2_seq_len] 第二句的0/1mask矩阵 + :return: torch.Tensor out_x1: [batch_size, x1_seq_len, hidden_size] 第一句attend到的特征表示 + torch.Tensor out_x2: [batch_size, x2_seq_len, hidden_size] 第一句attend到的特征表示 + """ assert in_x1.size()[0] == in_x2.size()[0] assert in_x1.size()[2] == in_x2.size()[2] diff --git a/fastNLP/modules/encoder/transformer.py b/fastNLP/modules/encoder/transformer.py index d7b8c544..d1262141 100644 --- a/fastNLP/modules/encoder/transformer.py +++ b/fastNLP/modules/encoder/transformer.py @@ -1,6 +1,6 @@ from torch import nn -from ..aggregator.attention import MultiHeadAtte +from ..aggregator.attention import MultiHeadAttention from ..dropout import TimestepDropout @@ -18,7 +18,7 @@ class TransformerEncoder(nn.Module): class SubLayer(nn.Module): def __init__(self, model_size, inner_size, key_size, value_size, num_head, dropout=0.1): super(TransformerEncoder.SubLayer, self).__init__() - self.atte = MultiHeadAtte(model_size, key_size, value_size, num_head, dropout) + self.atte = MultiHeadAttention(model_size, key_size, value_size, num_head, dropout) self.norm1 = nn.LayerNorm(model_size) self.ffn = nn.Sequential(nn.Linear(model_size, inner_size), nn.ReLU(), From 967e5e568389db8f98fa27c43c2c065470b307f3 Mon Sep 17 00:00:00 2001 From: ChenXin Date: Mon, 22 Apr 2019 01:31:41 +0800 Subject: [PATCH 2/9] doc tools --- docs/Makefile | 4 + docs/source/conf.py | 6 +- docs/source/fastNLP.api.rst | 52 +++++-- docs/source/fastNLP.core.rst | 98 ++++++++---- docs/source/fastNLP.io.rst | 48 +++--- docs/source/fastNLP.models.rst | 96 ++++++++++-- docs/source/fastNLP.modules.aggregator.rst | 42 ++++-- docs/source/fastNLP.modules.decoder.rst | 24 ++- docs/source/fastNLP.modules.encoder.rst | 74 ++++++--- docs/source/fastNLP.modules.rst | 33 ++++- docs/source/fastNLP.rst | 13 +- fastNLP/api/__init__.py | 3 + fastNLP/api/api.py | 26 ++-- fastNLP/automl/enas_trainer.py | 15 +- fastNLP/core/dataset.py | 2 +- fastNLP/core/fieldarray.py | 16 +- fastNLP/core/instance.py | 15 +- fastNLP/core/losses.py | 4 +- fastNLP/core/metrics.py | 165 ++++++++++++--------- fastNLP/core/trainer.py | 139 ++++++++--------- fastNLP/core/utils.py | 9 +- fastNLP/models/char_language_model.py | 13 +- fastNLP/models/enas_trainer.py | 15 +- 23 files changed, 599 insertions(+), 313 deletions(-) diff --git a/docs/Makefile b/docs/Makefile index e978dfe6..6a5c7375 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -3,6 +3,7 @@ # You can set these variables from the command line. SPHINXOPTS = +SPHINXAPIDOC = sphinx-apidoc SPHINXBUILD = sphinx-build SPHINXPROJ = fastNLP SOURCEDIR = source @@ -12,6 +13,9 @@ BUILDDIR = build help: @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) +apidoc: + @$(SPHINXAPIDOC) -f -o source ../fastNLP + .PHONY: help Makefile # Catch-all target: route all unknown targets to Sphinx using the new diff --git a/docs/source/conf.py b/docs/source/conf.py index e449a9f8..96f7f437 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -23,9 +23,9 @@ copyright = '2018, xpqiu' author = 'xpqiu' # The short X.Y version -version = '0.2' +version = '0.4' # The full version, including alpha/beta/rc tags -release = '0.2' +release = '0.4' # -- General configuration --------------------------------------------------- @@ -67,7 +67,7 @@ language = None # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path . -exclude_patterns = [] +exclude_patterns = ['modules.rst'] # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' diff --git a/docs/source/fastNLP.api.rst b/docs/source/fastNLP.api.rst index eb9192da..ee2413fb 100644 --- a/docs/source/fastNLP.api.rst +++ b/docs/source/fastNLP.api.rst @@ -1,36 +1,62 @@ -fastNLP.api -============ +fastNLP.api package +=================== -fastNLP.api.api ----------------- +Submodules +---------- + +fastNLP.api.api module +---------------------- .. automodule:: fastNLP.api.api :members: + :undoc-members: + :show-inheritance: -fastNLP.api.converter ----------------------- +fastNLP.api.converter module +---------------------------- .. automodule:: fastNLP.api.converter :members: + :undoc-members: + :show-inheritance: -fastNLP.api.model\_zoo ------------------------ +fastNLP.api.examples module +--------------------------- -.. automodule:: fastNLP.api.model_zoo +.. automodule:: fastNLP.api.examples :members: + :undoc-members: + :show-inheritance: -fastNLP.api.pipeline ---------------------- +fastNLP.api.pipeline module +--------------------------- .. automodule:: fastNLP.api.pipeline :members: + :undoc-members: + :show-inheritance: -fastNLP.api.processor ----------------------- +fastNLP.api.processor module +---------------------------- .. automodule:: fastNLP.api.processor :members: + :undoc-members: + :show-inheritance: + +fastNLP.api.utils module +------------------------ + +.. automodule:: fastNLP.api.utils + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- .. automodule:: fastNLP.api :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.core.rst b/docs/source/fastNLP.core.rst index b9f6c89f..79d26c76 100644 --- a/docs/source/fastNLP.core.rst +++ b/docs/source/fastNLP.core.rst @@ -1,84 +1,126 @@ -fastNLP.core -============= +fastNLP.core package +==================== -fastNLP.core.batch -------------------- +Submodules +---------- + +fastNLP.core.batch module +------------------------- .. automodule:: fastNLP.core.batch :members: + :undoc-members: + :show-inheritance: + +fastNLP.core.callback module +---------------------------- -fastNLP.core.dataset ---------------------- +.. automodule:: fastNLP.core.callback + :members: + :undoc-members: + :show-inheritance: + +fastNLP.core.dataset module +--------------------------- .. automodule:: fastNLP.core.dataset :members: + :undoc-members: + :show-inheritance: -fastNLP.core.fieldarray ------------------------- +fastNLP.core.fieldarray module +------------------------------ .. automodule:: fastNLP.core.fieldarray :members: + :undoc-members: + :show-inheritance: -fastNLP.core.instance ----------------------- +fastNLP.core.instance module +---------------------------- .. automodule:: fastNLP.core.instance :members: + :undoc-members: + :show-inheritance: -fastNLP.core.losses --------------------- +fastNLP.core.losses module +-------------------------- .. automodule:: fastNLP.core.losses :members: + :undoc-members: + :show-inheritance: -fastNLP.core.metrics ---------------------- +fastNLP.core.metrics module +--------------------------- .. automodule:: fastNLP.core.metrics :members: + :undoc-members: + :show-inheritance: -fastNLP.core.optimizer ------------------------ +fastNLP.core.optimizer module +----------------------------- .. automodule:: fastNLP.core.optimizer :members: + :undoc-members: + :show-inheritance: -fastNLP.core.predictor ------------------------ +fastNLP.core.predictor module +----------------------------- .. automodule:: fastNLP.core.predictor :members: + :undoc-members: + :show-inheritance: -fastNLP.core.sampler ---------------------- +fastNLP.core.sampler module +--------------------------- .. automodule:: fastNLP.core.sampler :members: + :undoc-members: + :show-inheritance: -fastNLP.core.tester --------------------- +fastNLP.core.tester module +-------------------------- .. automodule:: fastNLP.core.tester :members: + :undoc-members: + :show-inheritance: -fastNLP.core.trainer ---------------------- +fastNLP.core.trainer module +--------------------------- .. automodule:: fastNLP.core.trainer :members: + :undoc-members: + :show-inheritance: -fastNLP.core.utils -------------------- +fastNLP.core.utils module +------------------------- .. automodule:: fastNLP.core.utils :members: + :undoc-members: + :show-inheritance: -fastNLP.core.vocabulary ------------------------- +fastNLP.core.vocabulary module +------------------------------ .. automodule:: fastNLP.core.vocabulary :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- .. automodule:: fastNLP.core :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.io.rst b/docs/source/fastNLP.io.rst index d91e0d1c..bb30c5e7 100644 --- a/docs/source/fastNLP.io.rst +++ b/docs/source/fastNLP.io.rst @@ -1,42 +1,54 @@ -fastNLP.io -=========== +fastNLP.io package +================== -fastNLP.io.base\_loader ------------------------- +Submodules +---------- + +fastNLP.io.base\_loader module +------------------------------ .. automodule:: fastNLP.io.base_loader :members: + :undoc-members: + :show-inheritance: -fastNLP.io.config\_io ----------------------- +fastNLP.io.config\_io module +---------------------------- .. automodule:: fastNLP.io.config_io :members: + :undoc-members: + :show-inheritance: -fastNLP.io.dataset\_loader ---------------------------- +fastNLP.io.dataset\_loader module +--------------------------------- .. automodule:: fastNLP.io.dataset_loader :members: + :undoc-members: + :show-inheritance: -fastNLP.io.embed\_loader -------------------------- +fastNLP.io.embed\_loader module +------------------------------- .. automodule:: fastNLP.io.embed_loader :members: + :undoc-members: + :show-inheritance: -fastNLP.io.logger ------------------- - -.. automodule:: fastNLP.io.logger - :members: - -fastNLP.io.model\_io ---------------------- +fastNLP.io.model\_io module +--------------------------- .. automodule:: fastNLP.io.model_io :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- .. automodule:: fastNLP.io :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.models.rst b/docs/source/fastNLP.models.rst index 7452fdf6..3ebf9608 100644 --- a/docs/source/fastNLP.models.rst +++ b/docs/source/fastNLP.models.rst @@ -1,42 +1,110 @@ -fastNLP.models -=============== +fastNLP.models package +====================== -fastNLP.models.base\_model ---------------------------- +Submodules +---------- + +fastNLP.models.base\_model module +--------------------------------- .. automodule:: fastNLP.models.base_model :members: + :undoc-members: + :show-inheritance: + +fastNLP.models.bert module +-------------------------- -fastNLP.models.biaffine\_parser --------------------------------- +.. automodule:: fastNLP.models.bert + :members: + :undoc-members: + :show-inheritance: + +fastNLP.models.biaffine\_parser module +-------------------------------------- .. automodule:: fastNLP.models.biaffine_parser :members: + :undoc-members: + :show-inheritance: -fastNLP.models.char\_language\_model -------------------------------------- +fastNLP.models.char\_language\_model module +------------------------------------------- .. automodule:: fastNLP.models.char_language_model :members: + :undoc-members: + :show-inheritance: -fastNLP.models.cnn\_text\_classification ------------------------------------------ +fastNLP.models.cnn\_text\_classification module +----------------------------------------------- .. automodule:: fastNLP.models.cnn_text_classification :members: + :undoc-members: + :show-inheritance: + +fastNLP.models.enas\_controller module +-------------------------------------- + +.. automodule:: fastNLP.models.enas_controller + :members: + :undoc-members: + :show-inheritance: + +fastNLP.models.enas\_model module +--------------------------------- + +.. automodule:: fastNLP.models.enas_model + :members: + :undoc-members: + :show-inheritance: -fastNLP.models.sequence\_modeling ----------------------------------- +fastNLP.models.enas\_trainer module +----------------------------------- + +.. automodule:: fastNLP.models.enas_trainer + :members: + :undoc-members: + :show-inheritance: + +fastNLP.models.enas\_utils module +--------------------------------- + +.. automodule:: fastNLP.models.enas_utils + :members: + :undoc-members: + :show-inheritance: + +fastNLP.models.sequence\_modeling module +---------------------------------------- .. automodule:: fastNLP.models.sequence_modeling :members: + :undoc-members: + :show-inheritance: -fastNLP.models.snli --------------------- +fastNLP.models.snli module +-------------------------- .. automodule:: fastNLP.models.snli :members: + :undoc-members: + :show-inheritance: + +fastNLP.models.star\_transformer module +--------------------------------------- + +.. automodule:: fastNLP.models.star_transformer + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- .. automodule:: fastNLP.models :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.modules.aggregator.rst b/docs/source/fastNLP.modules.aggregator.rst index 073da4a5..63d351e4 100644 --- a/docs/source/fastNLP.modules.aggregator.rst +++ b/docs/source/fastNLP.modules.aggregator.rst @@ -1,36 +1,54 @@ -fastNLP.modules.aggregator -=========================== +fastNLP.modules.aggregator package +================================== -fastNLP.modules.aggregator.attention -------------------------------------- +Submodules +---------- + +fastNLP.modules.aggregator.attention module +------------------------------------------- .. automodule:: fastNLP.modules.aggregator.attention :members: + :undoc-members: + :show-inheritance: -fastNLP.modules.aggregator.avg\_pool -------------------------------------- +fastNLP.modules.aggregator.avg\_pool module +------------------------------------------- .. automodule:: fastNLP.modules.aggregator.avg_pool :members: + :undoc-members: + :show-inheritance: -fastNLP.modules.aggregator.kmax\_pool --------------------------------------- +fastNLP.modules.aggregator.kmax\_pool module +-------------------------------------------- .. automodule:: fastNLP.modules.aggregator.kmax_pool :members: + :undoc-members: + :show-inheritance: -fastNLP.modules.aggregator.max\_pool -------------------------------------- +fastNLP.modules.aggregator.max\_pool module +------------------------------------------- .. automodule:: fastNLP.modules.aggregator.max_pool :members: + :undoc-members: + :show-inheritance: -fastNLP.modules.aggregator.self\_attention -------------------------------------------- +fastNLP.modules.aggregator.self\_attention module +------------------------------------------------- .. automodule:: fastNLP.modules.aggregator.self_attention :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- .. automodule:: fastNLP.modules.aggregator :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.modules.decoder.rst b/docs/source/fastNLP.modules.decoder.rst index 6844543a..25602b2c 100644 --- a/docs/source/fastNLP.modules.decoder.rst +++ b/docs/source/fastNLP.modules.decoder.rst @@ -1,18 +1,30 @@ -fastNLP.modules.decoder -======================== +fastNLP.modules.decoder package +=============================== -fastNLP.modules.decoder.CRF ----------------------------- +Submodules +---------- + +fastNLP.modules.decoder.CRF module +---------------------------------- .. automodule:: fastNLP.modules.decoder.CRF :members: + :undoc-members: + :show-inheritance: -fastNLP.modules.decoder.MLP ----------------------------- +fastNLP.modules.decoder.MLP module +---------------------------------- .. automodule:: fastNLP.modules.decoder.MLP :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- .. automodule:: fastNLP.modules.decoder :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.modules.encoder.rst b/docs/source/fastNLP.modules.encoder.rst index ea8fc699..ab93a169 100644 --- a/docs/source/fastNLP.modules.encoder.rst +++ b/docs/source/fastNLP.modules.encoder.rst @@ -1,60 +1,94 @@ -fastNLP.modules.encoder -======================== +fastNLP.modules.encoder package +=============================== -fastNLP.modules.encoder.char\_embedding ----------------------------------------- +Submodules +---------- + +fastNLP.modules.encoder.char\_embedding module +---------------------------------------------- .. automodule:: fastNLP.modules.encoder.char_embedding :members: + :undoc-members: + :show-inheritance: -fastNLP.modules.encoder.conv ------------------------------ +fastNLP.modules.encoder.conv module +----------------------------------- .. automodule:: fastNLP.modules.encoder.conv :members: + :undoc-members: + :show-inheritance: -fastNLP.modules.encoder.conv\_maxpool --------------------------------------- +fastNLP.modules.encoder.conv\_maxpool module +-------------------------------------------- .. automodule:: fastNLP.modules.encoder.conv_maxpool :members: + :undoc-members: + :show-inheritance: -fastNLP.modules.encoder.embedding ----------------------------------- +fastNLP.modules.encoder.embedding module +---------------------------------------- .. automodule:: fastNLP.modules.encoder.embedding :members: + :undoc-members: + :show-inheritance: -fastNLP.modules.encoder.linear -------------------------------- +fastNLP.modules.encoder.linear module +------------------------------------- .. automodule:: fastNLP.modules.encoder.linear :members: + :undoc-members: + :show-inheritance: -fastNLP.modules.encoder.lstm ------------------------------ +fastNLP.modules.encoder.lstm module +----------------------------------- .. automodule:: fastNLP.modules.encoder.lstm :members: + :undoc-members: + :show-inheritance: -fastNLP.modules.encoder.masked\_rnn ------------------------------------- +fastNLP.modules.encoder.masked\_rnn module +------------------------------------------ .. automodule:: fastNLP.modules.encoder.masked_rnn :members: + :undoc-members: + :show-inheritance: -fastNLP.modules.encoder.transformer ------------------------------------- +fastNLP.modules.encoder.star\_transformer module +------------------------------------------------ + +.. automodule:: fastNLP.modules.encoder.star_transformer + :members: + :undoc-members: + :show-inheritance: + +fastNLP.modules.encoder.transformer module +------------------------------------------ .. automodule:: fastNLP.modules.encoder.transformer :members: + :undoc-members: + :show-inheritance: -fastNLP.modules.encoder.variational\_rnn ------------------------------------------ +fastNLP.modules.encoder.variational\_rnn module +----------------------------------------------- .. automodule:: fastNLP.modules.encoder.variational_rnn :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- .. automodule:: fastNLP.modules.encoder :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.modules.rst b/docs/source/fastNLP.modules.rst index 965fb27d..57858176 100644 --- a/docs/source/fastNLP.modules.rst +++ b/docs/source/fastNLP.modules.rst @@ -1,5 +1,8 @@ -fastNLP.modules -================ +fastNLP.modules package +======================= + +Subpackages +----------- .. toctree:: @@ -7,24 +10,38 @@ fastNLP.modules fastNLP.modules.decoder fastNLP.modules.encoder -fastNLP.modules.dropout ------------------------- +Submodules +---------- + +fastNLP.modules.dropout module +------------------------------ .. automodule:: fastNLP.modules.dropout :members: + :undoc-members: + :show-inheritance: -fastNLP.modules.other\_modules -------------------------------- +fastNLP.modules.other\_modules module +------------------------------------- .. automodule:: fastNLP.modules.other_modules :members: + :undoc-members: + :show-inheritance: -fastNLP.modules.utils ----------------------- +fastNLP.modules.utils module +---------------------------- .. automodule:: fastNLP.modules.utils :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- .. automodule:: fastNLP.modules :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.rst b/docs/source/fastNLP.rst index 61882359..6348c9a6 100644 --- a/docs/source/fastNLP.rst +++ b/docs/source/fastNLP.rst @@ -1,13 +1,22 @@ -fastNLP -======== +fastNLP package +=============== + +Subpackages +----------- .. toctree:: fastNLP.api + fastNLP.automl fastNLP.core fastNLP.io fastNLP.models fastNLP.modules +Module contents +--------------- + .. automodule:: fastNLP :members: + :undoc-members: + :show-inheritance: diff --git a/fastNLP/api/__init__.py b/fastNLP/api/__init__.py index a21a4c42..ae31b80b 100644 --- a/fastNLP/api/__init__.py +++ b/fastNLP/api/__init__.py @@ -1 +1,4 @@ +""" + 这是 API 部分的注释 +""" from .api import CWS, POS, Parser diff --git a/fastNLP/api/api.py b/fastNLP/api/api.py index 53a80131..b001629c 100644 --- a/fastNLP/api/api.py +++ b/fastNLP/api/api.py @@ -1,3 +1,7 @@ +""" +API.API 的文档 + +""" import warnings import torch @@ -184,17 +188,17 @@ class CWS(API): """ 传入一个分词文件路径，返回该数据集上分词f1, precision, recall。 分词文件应该为: - 1 编者按 编者按 NN O 11 nmod:topic - 2 ： ： PU O 11 punct - 3 7月 7月 NT DATE 4 compound:nn - 4 12日 12日 NT DATE 11 nmod:tmod - 5 ， ， PU O 11 punct - - 1 这 这 DT O 3 det - 2 款 款 M O 1 mark:clf - 3 飞行 飞行 NN O 8 nsubj - 4 从 从 P O 5 case - 5 外型 外型 NN O 8 nmod:prep + 1 编者按 编者按 NN O 11 nmod:topic + 2 ： ： PU O 11 punct + 3 7月 7月 NT DATE 4 compound:nn + 4 12日 12日 NT DATE 11 nmod:tmod + 5 ， ， PU O 11 punct + + 1 这 这 DT O 3 det + 2 款 款 M O 1 mark:clf + 3 飞行 飞行 NN O 8 nsubj + 4 从 从 P O 5 case + 5 外型 外型 NN O 8 nmod:prep 以空行分割两个句子，有内容的每行有7列。 :param filepath: str, 文件路径路径。 diff --git a/fastNLP/automl/enas_trainer.py b/fastNLP/automl/enas_trainer.py index 7c0da752..061d604c 100644 --- a/fastNLP/automl/enas_trainer.py +++ b/fastNLP/automl/enas_trainer.py @@ -62,13 +62,14 @@ class ENASTrainer(fastNLP.Trainer): """ :param bool load_best_model: 该参数只有在初始化提供了dev_data的情况下有效，如果True, trainer将在返回之前重新加载dev表现 最好的模型参数。 - :return results: 返回一个字典类型的数据, 内含以下内容:: - - seconds: float, 表示训练时长 - 以下三个内容只有在提供了dev_data的情况下会有。 - best_eval: Dict of Dict, 表示evaluation的结果 - best_epoch: int，在第几个epoch取得的最佳值 - best_step: int, 在第几个step(batch)更新取得的最佳值 + :return results: 返回一个字典类型的数据, + 内含以下内容:: + + seconds: float, 表示训练时长 + 以下三个内容只有在提供了dev_data的情况下会有。 + best_eval: Dict of Dict, 表示evaluation的结果 + best_epoch: int，在第几个epoch取得的最佳值 + best_step: int, 在第几个step(batch)更新取得的最佳值 """ results = {} diff --git a/fastNLP/core/dataset.py b/fastNLP/core/dataset.py index 76a34655..6cbfc20f 100644 --- a/fastNLP/core/dataset.py +++ b/fastNLP/core/dataset.py @@ -272,7 +272,7 @@ class DataSet(object): :param func: a function that takes an instance as input. :param str new_field_name: If not None, results of the function will be stored as a new field. - :param **kwargs: Accept parameters will be + :param kwargs: Accept parameters will be (1) is_input: boolean, will be ignored if new_field is None. If True, the new field will be as input. (2) is_target: boolean, will be ignored if new_field is None. If True, the new field will be as target. :return results: if new_field_name is not passed, returned values of the function over all instances. diff --git a/fastNLP/core/fieldarray.py b/fastNLP/core/fieldarray.py index 10fbbebe..caf2a1cf 100644 --- a/fastNLP/core/fieldarray.py +++ b/fastNLP/core/fieldarray.py @@ -48,12 +48,16 @@ class PadderBase: class AutoPadder(PadderBase): """ 根据contents的数据自动判定是否需要做padding。 - (1) 如果元素类型(元素类型是指field中最里层List的元素的数据类型, 可以通过FieldArray.dtype查看，比如['This', 'is', ...]的元素类 - 型为np.str, [[1,2], ...]的元素类型为np.int64)的数据不为(np.int64, np.float64)则不会进行padding - (2) 如果元素类型为(np.int64, np.float64), - (2.1) 如果该field的内容只有一个，比如为sequence_length, 则不进行padding - (2.2) 如果该field的内容为List, 那么会将Batch中的List pad为一样长。若该List下还有里层的List需要padding，请使用其它padder。 - 如果某个instance中field为[1, 2, 3]，则可以pad； 若为[[1,2], [3,4, ...]]则不能进行pad + + 1 如果元素类型(元素类型是指field中最里层List的元素的数据类型, 可以通过FieldArray.dtype查看，比如['This', 'is', ...]的元素类 + 型为np.str, [[1,2], ...]的元素类型为np.int64)的数据不为(np.int64, np.float64)则不会进行padding + + 2 如果元素类型为(np.int64, np.float64), + + 2.1 如果该field的内容只有一个，比如为sequence_length, 则不进行padding + + 2.2 如果该field的内容为List, 那么会将Batch中的List pad为一样长。若该List下还有里层的List需要padding，请使用其它padder。 + 如果某个instance中field为[1, 2, 3]，则可以pad； 若为[[1,2], [3,4, ...]]则不能进行pad """ def __init__(self, pad_val=0): """ diff --git a/fastNLP/core/instance.py b/fastNLP/core/instance.py index 5ac52e3f..fff992cc 100644 --- a/fastNLP/core/instance.py +++ b/fastNLP/core/instance.py @@ -1,13 +1,12 @@ class Instance(object): """An Instance is an example of data. - Example:: - ins = Instance(field_1=[1, 1, 1], field_2=[2, 2, 2]) - ins["field_1"] - >>[1, 1, 1] - ins.add_field("field_3", [3, 3, 3]) - - :param fields: a dict of (str: list). - + Example:: + + ins = Instance(field_1=[1, 1, 1], field_2=[2, 2, 2]) + ins["field_1"] + >>[1, 1, 1] + ins.add_field("field_3", [3, 3, 3]) + """ def __init__(self, **fields): diff --git a/fastNLP/core/losses.py b/fastNLP/core/losses.py index b52244e5..6b0b4460 100644 --- a/fastNLP/core/losses.py +++ b/fastNLP/core/losses.py @@ -272,7 +272,7 @@ def squash(predict, truth, **kwargs): :param predict: Tensor, model output :param truth: Tensor, truth from dataset - :param **kwargs: extra arguments + :param kwargs: extra arguments :return predict , truth: predict & truth after processing """ return predict.view(-1, predict.size()[-1]), truth.view(-1, ) @@ -316,7 +316,7 @@ def mask(predict, truth, **kwargs): :param predict: Tensor, [batch_size , max_len , tag_size] :param truth: Tensor, [batch_size , max_len] - :param **kwargs: extra arguments, kwargs["mask"]: ByteTensor, [batch_size , max_len], the mask Tensor. The position that is 1 will be selected. + :param kwargs: extra arguments, kwargs["mask"]: ByteTensor, [batch_size , max_len], the mask Tensor. The position that is 1 will be selected. :return predict , truth: predict & truth after processing """ diff --git a/fastNLP/core/metrics.py b/fastNLP/core/metrics.py index 5687cc85..314be0d9 100644 --- a/fastNLP/core/metrics.py +++ b/fastNLP/core/metrics.py @@ -17,66 +17,72 @@ class MetricBase(object): """Base class for all metrics. 所有的传入到Trainer, Tester的Metric需要继承自该对象。需要覆盖写入evaluate(), get_metric()方法。 + evaluate(xxx)中传入的是一个batch的数据。 + get_metric(xxx)当所有数据处理完毕，调用该方法得到最终的metric值 + 以分类问题中，Accuracy计算为例 - 假设model的forward返回dict中包含'pred'这个key, 并且该key需要用于Accuracy - class Model(nn.Module): - def __init__(xxx): - # do something - def forward(self, xxx): - # do something - return {'pred': pred, 'other_keys':xxx} # pred's shape: batch_size x num_classes + 假设model的forward返回dict中包含'pred'这个key, 并且该key需要用于Accuracy:: + + class Model(nn.Module): + def __init__(xxx): + # do something + def forward(self, xxx): + # do something + return {'pred': pred, 'other_keys':xxx} # pred's shape: batch_size x num_classes + 假设dataset中'label'这个field是需要预测的值，并且该field被设置为了target - 对应的AccMetric可以按如下的定义 - # version1, 只使用这一次 - class AccMetric(MetricBase): - def __init__(self): - super().__init__() - - # 根据你的情况自定义指标 - self.corr_num = 0 - self.total = 0 - - def evaluate(self, label, pred): # 这里的名称需要和dataset中target field与model返回的key是一样的，不然找不到对应的value - # dev或test时，每个batch结束会调用一次该方法，需要实现如何根据每个batch累加metric - self.total += label.size(0) - self.corr_num += label.eq(pred).sum().item() - - def get_metric(self, reset=True): # 在这里定义如何计算metric - acc = self.corr_num/self.total - if reset: # 是否清零以便重新计算 + 对应的AccMetric可以按如下的定义, version1, 只使用这一次:: + + class AccMetric(MetricBase): + def __init__(self): + super().__init__() + + # 根据你的情况自定义指标 self.corr_num = 0 self.total = 0 - return {'acc': acc} # 需要返回一个dict，key为该metric的名称，该名称会显示到Trainer的progress bar中 - - - # version2，如果需要复用Metric，比如下一次使用AccMetric时，dataset中目标field不叫label而叫y，或者model的输出不是pred - class AccMetric(MetricBase): - def __init__(self, label=None, pred=None): - # 假设在另一场景使用时，目标field叫y，model给出的key为pred_y。则只需要在初始化AccMetric时， - # acc_metric = AccMetric(label='y', pred='pred_y')即可。 - # 当初始化为acc_metric = AccMetric()，即label=None, pred=None, fastNLP会直接使用'label', 'pred'作为key去索取对 - # 应的的值 - super().__init__() - self._init_param_map(label=label, pred=pred) # 该方法会注册label和pred. 仅需要注册evaluate()方法会用到的参数名即可 - # 如果没有注册该则效果与version1就是一样的 - - # 根据你的情况自定义指标 - self.corr_num = 0 - self.total = 0 - - def evaluate(self, label, pred): # 这里的参数名称需要和self._init_param_map()注册时一致。 - # dev或test时，每个batch结束会调用一次该方法，需要实现如何根据每个batch累加metric - self.total += label.size(0) - self.corr_num += label.eq(pred).sum().item() - - def get_metric(self, reset=True): # 在这里定义如何计算metric - acc = self.corr_num/self.total - if reset: # 是否清零以便重新计算 + + def evaluate(self, label, pred): # 这里的名称需要和dataset中target field与model返回的key是一样的，不然找不到对应的value + # dev或test时，每个batch结束会调用一次该方法，需要实现如何根据每个batch累加metric + self.total += label.size(0) + self.corr_num += label.eq(pred).sum().item() + + def get_metric(self, reset=True): # 在这里定义如何计算metric + acc = self.corr_num/self.total + if reset: # 是否清零以便重新计算 + self.corr_num = 0 + self.total = 0 + return {'acc': acc} # 需要返回一个dict，key为该metric的名称，该名称会显示到Trainer的progress bar中 + + + version2，如果需要复用Metric，比如下一次使用AccMetric时，dataset中目标field不叫label而叫y，或者model的输出不是pred:: + + class AccMetric(MetricBase): + def __init__(self, label=None, pred=None): + # 假设在另一场景使用时，目标field叫y，model给出的key为pred_y。则只需要在初始化AccMetric时， + # acc_metric = AccMetric(label='y', pred='pred_y')即可。 + # 当初始化为acc_metric = AccMetric()，即label=None, pred=None, fastNLP会直接使用'label', 'pred'作为key去索取对 + # 应的的值 + super().__init__() + self._init_param_map(label=label, pred=pred) # 该方法会注册label和pred. 仅需要注册evaluate()方法会用到的参数名即可 + # 如果没有注册该则效果与version1就是一样的 + + # 根据你的情况自定义指标 self.corr_num = 0 self.total = 0 - return {'acc': acc} # 需要返回一个dict，key为该metric的名称，该名称会显示到Trainer的progress bar中 + + def evaluate(self, label, pred): # 这里的参数名称需要和self._init_param_map()注册时一致。 + # dev或test时，每个batch结束会调用一次该方法，需要实现如何根据每个batch累加metric + self.total += label.size(0) + self.corr_num += label.eq(pred).sum().item() + + def get_metric(self, reset=True): # 在这里定义如何计算metric + acc = self.corr_num/self.total + if reset: # 是否清零以便重新计算 + self.corr_num = 0 + self.total = 0 + return {'acc': acc} # 需要返回一个dict，key为该metric的名称，该名称会显示到Trainer的progress bar中 ``MetricBase`` handles validity check of its input dictionaries - ``pred_dict`` and ``target_dict``. @@ -84,12 +90,12 @@ class MetricBase(object): ``target_dict`` is the ground truth from DataSet where ``is_target`` is set ``True``. ``MetricBase`` will do the following type checks: - 1. whether self.evaluate has varargs, which is not supported. - 2. whether params needed by self.evaluate is not included in ``pred_dict``, ``target_dict``. - 3. whether params needed by self.evaluate duplicate in ``pred_dict``, ``target_dict``. + 1. whether self.evaluate has varargs, which is not supported. + 2. whether params needed by self.evaluate is not included in ``pred_dict``, ``target_dict``. + 3. whether params needed by self.evaluate duplicate in ``pred_dict``, ``target_dict``. Besides, before passing params into self.evaluate, this function will filter out params from output_dict and - target_dict which are not used in self.evaluate. (but if **kwargs presented in self.evaluate, no filtering + target_dict which are not used in self.evaluate. (but if kwargs presented in self.evaluate, no filtering will be conducted.) """ @@ -388,23 +394,26 @@ class SpanFPreRecMetric(MetricBase): """ 在序列标注问题中，以span的方式计算F, pre, rec. 比如中文Part of speech中，会以character的方式进行标注，句子'中国在亚洲'对应的POS可能为(以BMES为例) - ['B-NN', 'E-NN', 'S-DET', 'B-NN', 'E-NN']。该metric就是为类似情况下的F1计算。 - 最后得到的metric结果为 - { - 'f': xxx, # 这里使用f考虑以后可以计算f_beta值 - 'pre': xxx, - 'rec':xxx - } - 若only_gross=False, 即还会返回各个label的metric统计值 + ['B-NN', 'E-NN', 'S-DET', 'B-NN', 'E-NN']。该metric就是为类似情况下的F1计算。 + 最后得到的metric结果为:: + { - 'f': xxx, - 'pre': xxx, - 'rec':xxx, - 'f-label': xxx, - 'pre-label': xxx, - 'rec-label':xxx, - ... - } + 'f': xxx, # 这里使用f考虑以后可以计算f_beta值 + 'pre': xxx, + 'rec':xxx + } + + 若only_gross=False, 即还会返回各个label的metric统计值:: + + { + 'f': xxx, + 'pre': xxx, + 'rec':xxx, + 'f-label': xxx, + 'pre-label': xxx, + 'rec-label':xxx, + ... + } """ def __init__(self, tag_vocab, pred=None, target=None, seq_lens=None, encoding_type='bio', ignore_labels=None, @@ -573,13 +582,21 @@ class BMESF1PreRecMetric(MetricBase): """ 按照BMES标注方式计算f1, precision, recall。由于可能存在非法tag，比如"BS"，所以需要用以下的表格做转换，cur_B意思是当前tag是B， next_B意思是后一个tag是B。则cur_B=S，即将当前被predict是B的tag标为S；next_M=B, 即将后一个被predict是M的tag标为B + + +-------+---------+----------+----------+---------+---------+ | | next_B | next_M | next_E | next_S | end | - |:-----:|:-------:|:--------:|:--------:|:-------:|:-------:| - | start | 合法 | next_M=B | next_E=S | 合法 | - | + +=======+=========+==========+==========+=========+=========+ + | start | 合法 | next_M=B | next_E=S | 合法 | -- | + +-------+---------+----------+----------+---------+---------+ | cur_B | cur_B=S | 合法 | 合法 | cur_B=S | cur_B=S | + +-------+---------+----------+----------+---------+---------+ | cur_M | cur_M=E | 合法 | 合法 | cur_M=E | cur_M=E | + +-------+---------+----------+----------+---------+---------+ | cur_E | 合法 | next_M=B | next_E=S | 合法 | 合法 | + +-------+---------+----------+----------+---------+---------+ | cur_S | 合法 | next_M=B | next_E=S | 合法 | 合法 | + +-------+---------+----------+----------+---------+---------+ + 举例： prediction为BSEMS，会被认为是SSSSS. diff --git a/fastNLP/core/trainer.py b/fastNLP/core/trainer.py index b45dd148..250cfdb0 100644 --- a/fastNLP/core/trainer.py +++ b/fastNLP/core/trainer.py @@ -66,28 +66,28 @@ class Trainer(object): 不足，通过设置batch_size=32, update_every=4达到目的 """ super(Trainer, self).__init__() - + if not isinstance(train_data, DataSet): raise TypeError(f"The type of train_data must be fastNLP.DataSet, got {type(train_data)}.") if not isinstance(model, nn.Module): raise TypeError(f"The type of model must be torch.nn.Module, got {type(model)}.") - + # check metrics and dev_data if (not metrics) and dev_data is not None: raise ValueError("No metric for dev_data evaluation.") if metrics and (dev_data is None): raise ValueError("No dev_data for evaluations, pass dev_data or set metrics to None. ") - + # check update every - assert update_every>=1, "update_every must be no less than 1." + assert update_every >= 1, "update_every must be no less than 1." self.update_every = int(update_every) - + # check save_path if not (save_path is None or isinstance(save_path, str)): raise ValueError("save_path can only be None or `str`.") # prepare evaluate metrics = _prepare_metrics(metrics) - + # parse metric_key # increase_better is True. It means the exp result gets better if the indicator increases. # It is true by default. @@ -97,19 +97,19 @@ class Trainer(object): self.metric_key = metric_key[1:] if metric_key[0] == "+" or metric_key[0] == "-" else metric_key elif len(metrics) > 0: self.metric_key = metrics[0].__class__.__name__.lower().strip('metric') - + # prepare loss losser = _prepare_losser(loss) - + # sampler check if sampler is not None and not isinstance(sampler, BaseSampler): raise ValueError("The type of sampler should be fastNLP.BaseSampler, got {}.".format(type(sampler))) - + if check_code_level > -1: _check_code(dataset=train_data, model=model, losser=losser, metrics=metrics, dev_data=dev_data, metric_key=metric_key, check_level=check_code_level, batch_size=min(batch_size, DEFAULT_CHECK_BATCH_SIZE)) - + self.train_data = train_data self.dev_data = dev_data # If None, No validation. self.model = model @@ -120,7 +120,7 @@ class Trainer(object): self.use_cuda = bool(use_cuda) self.save_path = save_path self.print_every = int(print_every) - self.validate_every = int(validate_every) if validate_every!=0 else -1 + self.validate_every = int(validate_every) if validate_every != 0 else -1 self.best_metric_indicator = None self.best_dev_epoch = None self.best_dev_step = None @@ -129,19 +129,19 @@ class Trainer(object): self.prefetch = prefetch self.callback_manager = CallbackManager(env={"trainer": self}, callbacks=callbacks) self.n_steps = (len(self.train_data) // self.batch_size + int( - len(self.train_data) % self.batch_size != 0)) * self.n_epochs - + len(self.train_data) % self.batch_size != 0)) * self.n_epochs + if isinstance(optimizer, torch.optim.Optimizer): self.optimizer = optimizer else: if optimizer is None: optimizer = Adam(lr=0.01, weight_decay=0) self.optimizer = optimizer.construct_from_pytorch(self.model.parameters()) - + self.use_tqdm = use_tqdm self.pbar = None self.print_every = abs(self.print_every) - + if self.dev_data is not None: self.tester = Tester(model=self.model, data=self.dev_data, @@ -149,14 +149,13 @@ class Trainer(object): batch_size=self.batch_size, use_cuda=self.use_cuda, verbose=0) - + self.step = 0 self.start_time = None # start timestamp - + self.callback_manager = CallbackManager(env={"trainer": self}, callbacks=callbacks) - - + def train(self, load_best_model=True): """ @@ -185,14 +184,15 @@ class Trainer(object): 根据metrics进行evaluation，并根据是否提供了save_path判断是否存储模型 :param bool load_best_model: 该参数只有在初始化提供了dev_data的情况下有效，如果True, trainer将在返回之前重新加载dev表现 - 最好的模型参数。 - :return results: 返回一个字典类型的数据, 内含以下内容:: + 最好的模型参数。 + :return results: 返回一个字典类型的数据, + 内含以下内容:: - seconds: float, 表示训练时长 - 以下三个内容只有在提供了dev_data的情况下会有。 - best_eval: Dict of Dict, 表示evaluation的结果 - best_epoch: int，在第几个epoch取得的最佳值 - best_step: int, 在第几个step(batch)更新取得的最佳值 + seconds: float, 表示训练时长 + 以下三个内容只有在提供了dev_data的情况下会有。 + best_eval: Dict of Dict, 表示evaluation的结果 + best_epoch: int，在第几个epoch取得的最佳值 + best_step: int, 在第几个step(batch)更新取得的最佳值 """ results = {} @@ -205,21 +205,22 @@ class Trainer(object): self.model = self.model.cuda() self._model_device = self.model.parameters().__next__().device self._mode(self.model, is_test=False) - + self.start_time = str(datetime.now().strftime('%Y-%m-%d-%H-%M-%S')) start_time = time.time() print("training epochs started " + self.start_time, flush=True) - + try: self.callback_manager.on_train_begin() self._train() self.callback_manager.on_train_end() except (CallbackException, KeyboardInterrupt) as e: self.callback_manager.on_exception(e) - + if self.dev_data is not None and hasattr(self, 'best_dev_perf'): - print("\nIn Epoch:{}/Step:{}, got best dev performance:".format(self.best_dev_epoch, self.best_dev_step) + - self.tester._format_eval_results(self.best_dev_perf),) + print( + "\nIn Epoch:{}/Step:{}, got best dev performance:".format(self.best_dev_epoch, self.best_dev_step) + + self.tester._format_eval_results(self.best_dev_perf), ) results['best_eval'] = self.best_dev_perf results['best_epoch'] = self.best_dev_epoch results['best_step'] = self.best_dev_step @@ -233,9 +234,9 @@ class Trainer(object): finally: pass results['seconds'] = round(time.time() - start_time, 2) - + return results - + def _train(self): if not self.use_tqdm: from fastNLP.core.utils import pseudo_tqdm as inner_tqdm @@ -244,13 +245,13 @@ class Trainer(object): self.step = 0 self.epoch = 0 start = time.time() - + with inner_tqdm(total=self.n_steps, postfix='loss:{0:<6.5f}', leave=False, dynamic_ncols=True) as pbar: self.pbar = pbar if isinstance(pbar, tqdm) else None avg_loss = 0 data_iterator = Batch(self.train_data, batch_size=self.batch_size, sampler=self.sampler, as_numpy=False, prefetch=self.prefetch) - for epoch in range(1, self.n_epochs+1): + for epoch in range(1, self.n_epochs + 1): self.epoch = epoch pbar.set_description_str(desc="Epoch {}/{}".format(epoch, self.n_epochs)) # early stopping @@ -262,22 +263,22 @@ class Trainer(object): # negative sampling; replace unknown; re-weight batch_y self.callback_manager.on_batch_begin(batch_x, batch_y, indices) prediction = self._data_forward(self.model, batch_x) - + # edit prediction self.callback_manager.on_loss_begin(batch_y, prediction) loss = self._compute_loss(prediction, batch_y).mean() avg_loss += loss.item() - loss = loss/self.update_every - + loss = loss / self.update_every + # Is loss NaN or inf? requires_grad = False self.callback_manager.on_backward_begin(loss) self._grad_backward(loss) self.callback_manager.on_backward_end() - + self._update() self.callback_manager.on_step_end() - - if (self.step+1) % self.print_every == 0: + + if (self.step + 1) % self.print_every == 0: avg_loss = avg_loss / self.print_every if self.use_tqdm: print_output = "loss:{0:<6.5f}".format(avg_loss) @@ -290,34 +291,34 @@ class Trainer(object): pbar.set_postfix_str(print_output) avg_loss = 0 self.callback_manager.on_batch_end() - + if ((self.validate_every > 0 and self.step % self.validate_every == 0) or (self.validate_every < 0 and self.step % len(data_iterator) == 0)) \ and self.dev_data is not None: eval_res = self._do_validation(epoch=epoch, step=self.step) eval_str = "Evaluation at Epoch {}/{}. Step:{}/{}. ".format(epoch, self.n_epochs, self.step, self.n_steps) + \ - self.tester._format_eval_results(eval_res) + self.tester._format_eval_results(eval_res) pbar.write(eval_str + '\n') - + # ================= mini-batch end ==================== # - + # lr decay; early stopping self.callback_manager.on_epoch_end() # =============== epochs end =================== # pbar.close() self.pbar = None # ============ tqdm end ============== # - + def _do_validation(self, epoch, step): self.callback_manager.on_valid_begin() res = self.tester.test() - + is_better_eval = False if self._better_eval_result(res): if self.save_path is not None: self._save_model(self.model, - "best_" + "_".join([self.model.__class__.__name__, self.metric_key, self.start_time])) + "best_" + "_".join([self.model.__class__.__name__, self.metric_key, self.start_time])) else: self._best_model_states = {name: param.cpu().clone() for name, param in self.model.named_parameters()} self.best_dev_perf = res @@ -327,7 +328,7 @@ class Trainer(object): # get validation results; adjust optimizer self.callback_manager.on_valid_end(res, self.metric_key, self.optimizer, is_better_eval) return res - + def _mode(self, model, is_test=False): """Train mode or Test mode. This is for PyTorch currently. @@ -339,21 +340,21 @@ class Trainer(object): model.eval() else: model.train() - + def _update(self): """Perform weight update on a model. """ - if (self.step+1)%self.update_every==0: + if (self.step + 1) % self.update_every == 0: self.optimizer.step() - + def _data_forward(self, network, x): x = _build_args(network.forward, **x) y = network(**x) if not isinstance(y, dict): raise TypeError(f"The return value of {get_func_signature(network.forward)} should be dict, got {type(y)}.") return y - + def _grad_backward(self, loss): """Compute gradient with link rules. @@ -361,10 +362,10 @@ class Trainer(object): For PyTorch, just do "loss.backward()" """ - if self.step%self.update_every==0: + if self.step % self.update_every == 0: self.model.zero_grad() loss.backward() - + def _compute_loss(self, predict, truth): """Compute loss given prediction and ground truth. @@ -373,7 +374,7 @@ class Trainer(object): :return: a scalar """ return self.losser(predict, truth) - + def _save_model(self, model, model_name, only_param=False): """ 存储不含有显卡信息的state_dict或model :param model: @@ -394,7 +395,7 @@ class Trainer(object): model.cpu() torch.save(model, model_path) model.to(self._model_device) - + def _load_model(self, model, model_name, only_param=False): # 返回bool值指示是否成功reload模型 if self.save_path is not None: @@ -409,7 +410,7 @@ class Trainer(object): else: return False return True - + def _better_eval_result(self, metrics): """Check if the current epoch yields better validation results. @@ -437,6 +438,7 @@ class Trainer(object): DEFAULT_CHECK_BATCH_SIZE = 2 DEFAULT_CHECK_NUM_BATCH = 2 + def _get_value_info(_dict): # given a dict value, return information about this dict's value. Return list of str strs = [] @@ -453,27 +455,28 @@ def _get_value_info(_dict): strs.append(_str) return strs + def _check_code(dataset, model, losser, metrics, batch_size=DEFAULT_CHECK_BATCH_SIZE, dev_data=None, metric_key=None, check_level=0): # check get_loss 方法 model_devcie = model.parameters().__next__().device - + batch = Batch(dataset=dataset, batch_size=batch_size, sampler=SequentialSampler()) for batch_count, (batch_x, batch_y) in enumerate(batch): _move_dict_value_to_device(batch_x, batch_y, device=model_devcie) # forward check - if batch_count==0: + if batch_count == 0: info_str = "" input_fields = _get_value_info(batch_x) target_fields = _get_value_info(batch_y) - if len(input_fields)>0: + if len(input_fields) > 0: info_str += "input fields after batch(if batch size is {}):\n".format(batch_size) info_str += "\n".join(input_fields) info_str += '\n' else: raise RuntimeError("There is no input field.") - if len(target_fields)>0: + if len(target_fields) > 0: info_str += "target fields after batch(if batch size is {}):\n".format(batch_size) info_str += "\n".join(target_fields) info_str += '\n' @@ -481,14 +484,14 @@ def _check_code(dataset, model, losser, metrics, batch_size=DEFAULT_CHECK_BATCH_ info_str += 'There is no target field.' print(info_str) _check_forward_error(forward_func=model.forward, dataset=dataset, - batch_x=batch_x, check_level=check_level) - + batch_x=batch_x, check_level=check_level) + refined_batch_x = _build_args(model.forward, **batch_x) pred_dict = model(**refined_batch_x) func_signature = get_func_signature(model.forward) if not isinstance(pred_dict, dict): raise TypeError(f"The return value of {func_signature} should be `dict`, not `{type(pred_dict)}`.") - + # loss check try: loss = losser(pred_dict, batch_y) @@ -512,7 +515,7 @@ def _check_code(dataset, model, losser, metrics, batch_size=DEFAULT_CHECK_BATCH_ model.zero_grad() if batch_count + 1 >= DEFAULT_CHECK_NUM_BATCH: break - + if dev_data is not None: tester = Tester(data=dev_data[:batch_size * DEFAULT_CHECK_NUM_BATCH], model=model, metrics=metrics, batch_size=batch_size, verbose=-1) @@ -526,7 +529,7 @@ def _check_eval_results(metrics, metric_key, metric_list): # metric_list: 多个用来做评价的指标，来自Trainer的初始化 if isinstance(metrics, tuple): loss, metrics = metrics - + if isinstance(metrics, dict): if len(metrics) == 1: # only single metric, just use it @@ -537,7 +540,7 @@ def _check_eval_results(metrics, metric_key, metric_list): if metrics_name not in metrics: raise RuntimeError(f"{metrics_name} is chosen to do validation, but got {metrics}") metric_dict = metrics[metrics_name] - + if len(metric_dict) == 1: indicator_val, indicator = list(metric_dict.values())[0], list(metric_dict.keys())[0] elif len(metric_dict) > 1 and metric_key is None: diff --git a/fastNLP/core/utils.py b/fastNLP/core/utils.py index d9141412..fc15166e 100644 --- a/fastNLP/core/utils.py +++ b/fastNLP/core/utils.py @@ -197,17 +197,22 @@ def get_func_signature(func): Given a function or method, return its signature. For example: - (1) function + + 1 function:: + def func(a, b='a', *args): xxxx get_func_signature(func) # 'func(a, b='a', *args)' - (2) method + + 2 method:: + class Demo: def __init__(self): xxx def forward(self, a, b='a', **args) demo = Demo() get_func_signature(demo.forward) # 'Demo.forward(self, a, b='a', **args)' + :param func: a function or a method :return: str or None """ diff --git a/fastNLP/models/char_language_model.py b/fastNLP/models/char_language_model.py index 5fbde3cc..d5e3359d 100644 --- a/fastNLP/models/char_language_model.py +++ b/fastNLP/models/char_language_model.py @@ -20,16 +20,23 @@ class Highway(nn.Module): class CharLM(nn.Module): """CNN + highway network + LSTM - # Input: + + # Input:: + 4D tensor with shape [batch_size, in_channel, height, width] - # Output: + + # Output:: + 2D Tensor with shape [batch_size, vocab_size] - # Arguments: + + # Arguments:: + char_emb_dim: the size of each character's attention word_emb_dim: the size of each word's attention vocab_size: num of unique words num_char: num of characters use_gpu: True or False + """ def __init__(self, char_emb_dim, word_emb_dim, diff --git a/fastNLP/models/enas_trainer.py b/fastNLP/models/enas_trainer.py index 6b51c897..26b7cd49 100644 --- a/fastNLP/models/enas_trainer.py +++ b/fastNLP/models/enas_trainer.py @@ -65,13 +65,14 @@ class ENASTrainer(fastNLP.Trainer): """ :param bool load_best_model: 该参数只有在初始化提供了dev_data的情况下有效，如果True, trainer将在返回之前重新加载dev表现 最好的模型参数。 - :return results: 返回一个字典类型的数据, 内含以下内容:: - - seconds: float, 表示训练时长 - 以下三个内容只有在提供了dev_data的情况下会有。 - best_eval: Dict of Dict, 表示evaluation的结果 - best_epoch: int，在第几个epoch取得的最佳值 - best_step: int, 在第几个step(batch)更新取得的最佳值 + :return results: 返回一个字典类型的数据, + 内含以下内容:: + + seconds: float, 表示训练时长 + 以下三个内容只有在提供了dev_data的情况下会有。 + best_eval: Dict of Dict, 表示evaluation的结果 + best_epoch: int，在第几个epoch取得的最佳值 + best_step: int, 在第几个step(batch)更新取得的最佳值 """ results = {} From 13d8978953026bcb6fb4046c7f6e0ce500458efb Mon Sep 17 00:00:00 2001 From: ChenXin Date: Mon, 22 Apr 2019 01:49:44 +0800 Subject: [PATCH 3/9] fix some doc errors --- fastNLP/io/config_io.py | 8 +++--- fastNLP/io/dataset_loader.py | 47 +++++++++++++++++++++--------------- fastNLP/io/embed_loader.py | 2 +- fastNLP/io/model_io.py | 12 +++++---- 4 files changed, 39 insertions(+), 30 deletions(-) diff --git a/fastNLP/io/config_io.py b/fastNLP/io/config_io.py index 5a64b96c..c0ffe53e 100644 --- a/fastNLP/io/config_io.py +++ b/fastNLP/io/config_io.py @@ -26,10 +26,10 @@ class ConfigLoader(BaseLoader): :param str file_path: the path of config file :param dict sections: the dict of ``{section_name(string): ConfigSection object}`` - Example:: - - test_args = ConfigSection() - ConfigLoader("config.cfg").load_config("./data_for_tests/config", {"POS_test": test_args}) + Example:: + + test_args = ConfigSection() + ConfigLoader("config.cfg").load_config("./data_for_tests/config", {"POS_test": test_args}) """ assert isinstance(sections, dict) diff --git a/fastNLP/io/dataset_loader.py b/fastNLP/io/dataset_loader.py index e33384a8..87127cf8 100644 --- a/fastNLP/io/dataset_loader.py +++ b/fastNLP/io/dataset_loader.py @@ -9,7 +9,7 @@ from fastNLP.io.base_loader import DataLoaderRegister def convert_seq_dataset(data): """Create an DataSet instance that contains no labels. - :param data: list of list of strings, [num_examples, *]. + :param data: list of list of strings, [num_examples, \*]. Example:: [ @@ -28,7 +28,7 @@ def convert_seq_dataset(data): def convert_seq2tag_dataset(data): """Convert list of data into DataSet. - :param data: list of list of strings, [num_examples, *]. + :param data: list of list of strings, [num_examples, \*]. Example:: [ @@ -48,7 +48,7 @@ def convert_seq2tag_dataset(data): def convert_seq2seq_dataset(data): """Convert list of data into DataSet. - :param data: list of list of strings, [num_examples, *]. + :param data: list of list of strings, [num_examples, \*]. Example:: [ @@ -177,18 +177,18 @@ DataLoaderRegister.set_reader(RawDataSetLoader, 'read_rawdata') class DummyPOSReader(DataSetLoader): """A simple reader for a dummy POS tagging dataset. - In these datasets, each line are divided by "\t". The first Col is the vocabulary and the second + In these datasets, each line are divided by "\\\\t". The first Col is the vocabulary and the second Col is the label. Different sentence are divided by an empty line. - E.g:: + E.g:: - Tom label1 - and label2 - Jerry label1 - . label3 - (separated by an empty line) - Hello label4 - world label5 - ! label3 + Tom label1 + and label2 + Jerry label1 + . label3 + (separated by an empty line) + Hello label4 + world label5 + ! label3 In this example, there are two sentences "Tom and Jerry ." and "Hello world !". Each word has its own label. """ @@ -200,11 +200,13 @@ class DummyPOSReader(DataSetLoader): """ :return data: three-level list Example:: + [ [ [word_11, word_12, ...], [label_1, label_1, ...] ], [ [word_21, word_22, ...], [label_2, label_1, ...] ], ... ] + """ with open(data_path, "r", encoding="utf-8") as f: lines = f.readlines() @@ -550,6 +552,7 @@ class SNLIDataSetReader(DataSetLoader): :param data: A 3D tensor. Example:: + [ [ [premise_word_11, premise_word_12, ...], [hypothesis_word_11, hypothesis_word_12, ...], [label_1] ], [ [premise_word_21, premise_word_22, ...], [hypothesis_word_21, hypothesis_word_22, ...], [label_2] ], @@ -647,7 +650,7 @@ class NaiveCWSReader(DataSetLoader): 例如:: 这是 fastNLP , 一个 非常 good 的 包 . - + 或者,即每个part后面还有一个pos tag 例如:: @@ -661,12 +664,15 @@ class NaiveCWSReader(DataSetLoader): def load(self, filepath, in_word_splitter=None, cut_long_sent=False): """ - 允许使用的情况有(默认以\t或空格作为seg) + 允许使用的情况有(默认以\\\\t或空格作为seg):: + 这是 fastNLP , 一个 非常 good 的 包 . - 和 + + 和:: + 也/D 在/P 團員/Na 之中/Ng ，/COMMACATEGORY + 如果splitter不为None则认为是第二种情况, 且我们会按splitter分割"也/D", 然后取第一部分. 例如"也/D".split('/')[0] - :param filepath: :param in_word_splitter: :param cut_long_sent: @@ -737,11 +743,12 @@ class ZhConllPOSReader(object): def load(self, path): """ - 返回的DataSet, 包含以下的field + 返回的DataSet, 包含以下的field:: + words：list of str, tag: list of str, 被加入了BMES tag, 比如原来的序列为['VP', 'NN', 'NN', ..]，会被认为是["S-VP", "B-NN", "M-NN",..] - 假定了输入为conll的格式，以空行隔开两个句子，每行共7列，即 - :: + + 假定了输入为conll的格式，以空行隔开两个句子，每行共7列，即:: 1 编者按 编者按 NN O 11 nmod:topic 2 ： ： PU O 11 punct diff --git a/fastNLP/io/embed_loader.py b/fastNLP/io/embed_loader.py index 5ad27c53..16ea0339 100644 --- a/fastNLP/io/embed_loader.py +++ b/fastNLP/io/embed_loader.py @@ -132,7 +132,7 @@ class EmbedLoader(BaseLoader): def load_with_vocab(embed_filepath, vocab, dtype=np.float32, normalize=True, error='ignore'): """ load pretraining embedding in {embed_file} based on words in vocab. Words in vocab but not in the pretraining - embedding are initialized from a normal distribution which has the mean and std of the found words vectors. + embedding are initialized from a normal distribution which has the mean and std of the found words vectors. The embedding type is determined automatically, support glove and word2vec(the first line only has two elements). :param embed_filepath: str, where to read pretrain embedding diff --git a/fastNLP/io/model_io.py b/fastNLP/io/model_io.py index 422eb919..53bdc7ce 100644 --- a/fastNLP/io/model_io.py +++ b/fastNLP/io/model_io.py @@ -31,16 +31,18 @@ class ModelLoader(BaseLoader): class ModelSaver(object): """Save a model + Example:: - :param str save_path: the path to the saving directory. - Example:: - - saver = ModelSaver("./save/model_ckpt_100.pkl") - saver.save_pytorch(model) + saver = ModelSaver("./save/model_ckpt_100.pkl") + saver.save_pytorch(model) """ def __init__(self, save_path): + """ + + :param save_path: the path to the saving directory. + """ self.save_path = save_path def save_pytorch(self, model, param_only=True): From c520d350827cda3415d2bfd0b033ed79e02ff352 Mon Sep 17 00:00:00 2001 From: yh Date: Mon, 22 Apr 2019 10:52:01 +0800 Subject: [PATCH 4/9] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=B8=BA=E4=B8=AD?= =?UTF-8?q?=E6=96=87=E6=B3=A8=E9=87=8A=EF=BC=8C=E5=A2=9E=E5=8A=A0viterbi?= =?UTF-8?q?=E8=A7=A3=E7=A0=81=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/core/dataset.py | 168 +++++++++++------- fastNLP/core/fieldarray.py | 22 +-- fastNLP/core/utils.py | 2 +- fastNLP/models/sequence_modeling.py | 2 +- fastNLP/modules/decoder/CRF.py | 113 ++++++------ fastNLP/modules/decoder/utils.py | 70 ++++++++ .../models/cws_model.py | 4 +- .../models/cws_transformer.py | 4 +- 8 files changed, 249 insertions(+), 136 deletions(-) create mode 100644 fastNLP/modules/decoder/utils.py diff --git a/fastNLP/core/dataset.py b/fastNLP/core/dataset.py index 76a34655..f0e27b83 100644 --- a/fastNLP/core/dataset.py +++ b/fastNLP/core/dataset.py @@ -151,16 +151,19 @@ class DataSet(object): assert name in self.field_arrays self.field_arrays[name].append(field) - def add_field(self, name, fields, padder=AutoPadder(pad_val=0), is_input=False, is_target=False, ignore_type=False): + def add_field(self, name, fields, padder=None, is_input=False, is_target=False, ignore_type=False): """Add a new field to the DataSet. :param str name: the name of the field. :param fields: a list of int, float, or other objects. - :param int padder: PadBase对象，如何对该Field进行padding。大部分情况使用默认值即可 + :param padder: PadBase对象，如何对该Field进行padding。如果为None则使用 :param bool is_input: whether this field is model input. :param bool is_target: whether this field is label or target. :param bool ignore_type: If True, do not perform type check. (Default: False) """ + if padder is None: + padder = AutoPadder(pad_val=0) + if len(self.field_arrays) != 0: if len(self) != len(fields): raise RuntimeError(f"The field to append must have the same size as dataset. " @@ -231,8 +234,8 @@ class DataSet(object): raise KeyError("{} is not a valid field name.".format(name)) def set_padder(self, field_name, padder): - """ - 为field_name设置padder + """为field_name设置padder + :param field_name: str, 设置field的padding方式为padder :param padder: PadderBase类型或None. 设置为None即删除padder。即对该field不进行padding操作. :return: @@ -242,8 +245,7 @@ class DataSet(object): self.field_arrays[field_name].set_padder(padder) def set_pad_val(self, field_name, pad_val): - """ - 为某个 + """为某个field设置对应的pad_val. :param field_name: str，修改该field的pad_val :param pad_val: int，该field的padder会以pad_val作为padding index @@ -254,43 +256,60 @@ class DataSet(object): self.field_arrays[field_name].set_pad_val(pad_val) def get_input_name(self): - """Get all field names with `is_input` as True. + """返回所有is_input被设置为True的field名称 - :return field_names: a list of str + :return list, 里面的元素为被设置为input的field名称 """ return [name for name, field in self.field_arrays.items() if field.is_input] def get_target_name(self): - """Get all field names with `is_target` as True. + """返回所有is_target被设置为True的field名称 - :return field_names: a list of str + :return list, 里面的元素为被设置为target的field名称 """ return [name for name, field in self.field_arrays.items() if field.is_target] - def apply(self, func, new_field_name=None, **kwargs): - """Apply a function to every instance of the DataSet. - - :param func: a function that takes an instance as input. - :param str new_field_name: If not None, results of the function will be stored as a new field. - :param **kwargs: Accept parameters will be - (1) is_input: boolean, will be ignored if new_field is None. If True, the new field will be as input. - (2) is_target: boolean, will be ignored if new_field is None. If True, the new field will be as target. - :return results: if new_field_name is not passed, returned values of the function over all instances. + def apply_field(self, func, field_name, new_field_name=None, **kwargs): + """将DataSet中的每个instance中的`field_name`这个field传给func，并获取它的返回值. + + :param func: Callable, input是instance的`field_name`这个field. + :param field_name: str, 传入func的是哪个field. + :param new_field_name: (str, None). 如果不是None，将func的返回值放入这个名为`new_field_name`的新field中，如果名称与已有 + 的field相同，则覆盖之前的field. + :param **kwargs: 合法的参数有以下三个 + (1) is_input: bool, 如果为True则将`new_field_name`这个field设置为input + (2) is_target: bool, 如果为True则将`new_field_name`这个field设置为target + (3) ignore_type: bool, 如果为True则将`new_field_name`这个field的ignore_type设置为true, 忽略其类型 + :return: List[], 里面的元素为func的返回值，所以list长度为DataSet的长度 """ - assert len(self)!=0, "Null dataset cannot use .apply()." + assert len(self)!=0, "Null DataSet cannot use apply()." + if field_name not in self: + raise KeyError("DataSet has no field named `{}`.".format(field_name)) results = [] idx = -1 try: for idx, ins in enumerate(self._inner_iter()): - results.append(func(ins)) + results.append(func(ins[field_name])) except Exception as e: if idx!=-1: print("Exception happens at the `{}`th instance.".format(idx)) raise e - # results = [func(ins) for ins in self._inner_iter()] if not (new_field_name is None) and len(list(filter(lambda x: x is not None, results))) == 0: # all None raise ValueError("{} always return None.".format(get_func_signature(func=func))) + if new_field_name is not None: + self._add_apply_field(results, new_field_name, kwargs) + + return results + + def _add_apply_field(self, results, new_field_name, kwargs): + """将results作为加入到新的field中，field名称为new_field_name + + :param results: List[], 一般是apply*()之后的结果 + :param new_field_name: str, 新加入的field的名称 + :param kwargs: dict, 用户apply*()时传入的自定义参数 + :return: + """ extra_param = {} if 'is_input' in kwargs: extra_param['is_input'] = kwargs['is_input'] @@ -298,56 +317,84 @@ class DataSet(object): extra_param['is_target'] = kwargs['is_target'] if 'ignore_type' in kwargs: extra_param['ignore_type'] = kwargs['ignore_type'] - if new_field_name is not None: - if new_field_name in self.field_arrays: - # overwrite the field, keep same attributes - old_field = self.field_arrays[new_field_name] - if 'is_input' not in extra_param: - extra_param['is_input'] = old_field.is_input - if 'is_target' not in extra_param: - extra_param['is_target'] = old_field.is_target - if 'ignore_type' not in extra_param: - extra_param['ignore_type'] = old_field.ignore_type - self.add_field(name=new_field_name, fields=results, is_input=extra_param["is_input"], - is_target=extra_param["is_target"], ignore_type=extra_param['ignore_type']) - else: - self.add_field(name=new_field_name, fields=results, is_input=extra_param.get("is_input", None), - is_target=extra_param.get("is_target", None), - ignore_type=extra_param.get("ignore_type", False)) + if new_field_name in self.field_arrays: + # overwrite the field, keep same attributes + old_field = self.field_arrays[new_field_name] + if 'is_input' not in extra_param: + extra_param['is_input'] = old_field.is_input + if 'is_target' not in extra_param: + extra_param['is_target'] = old_field.is_target + if 'ignore_type' not in extra_param: + extra_param['ignore_type'] = old_field.ignore_type + self.add_field(name=new_field_name, fields=results, is_input=extra_param["is_input"], + is_target=extra_param["is_target"], ignore_type=extra_param['ignore_type']) else: - return results + self.add_field(name=new_field_name, fields=results, is_input=extra_param.get("is_input", None), + is_target=extra_param.get("is_target", None), + ignore_type=extra_param.get("ignore_type", False)) + + def apply(self, func, new_field_name=None, **kwargs): + """将DataSet中每个instance传入到func中，并获取它的返回值. + + :param func: Callable, 参数是DataSet中的instance + :param new_field_name: (None, str). (1) None, 不创建新的field; (2) str，将func的返回值放入这个名为 + `new_field_name`的新field中，如果名称与已有的field相同，则覆盖之前的field; + :param kwargs: 合法的参数有以下三个 + (1) is_input: bool, 如果为True则将`new_field_name`的field设置为input + (2) is_target: bool, 如果为True则将`new_field_name`的field设置为target + (3) ignore_type: bool, 如果为True则将`new_field_name`的field的ignore_type设置为true, 忽略其类型 + :return: List[], 里面的元素为func的返回值，所以list长度为DataSet的长度 + """ + assert len(self)!=0, "Null DataSet cannot use apply()." + idx = -1 + try: + results = [] + for idx, ins in enumerate(self._inner_iter()): + results.append(func(ins)) + except Exception as e: + if idx!=-1: + print("Exception happens at the `{}`th instance.".format(idx)) + raise e + # results = [func(ins) for ins in self._inner_iter()] + if not (new_field_name is None) and len(list(filter(lambda x: x is not None, results))) == 0: # all None + raise ValueError("{} always return None.".format(get_func_signature(func=func))) + + if new_field_name is not None: + self._add_apply_field(results, new_field_name, kwargs) + + return results def drop(self, func, inplace=True): - """Drop instances if a condition holds. + """func接受一个instance，返回bool值，返回值为True时，该instance会被删除。 - :param func: a function that takes an Instance object as input, and returns bool. - The instance will be dropped if the function returns True. - :param inplace: bool, whether to drop inpalce. Otherwise a new dataset will be returned. + :param func: Callable, 接受一个instance作为参数，返回bool值。为True时删除该instance + :param inplace: bool, 是否在当前DataSet中直接删除instance。如果为False，返回值为一个删除了相应instance的新的DataSet + :return: DataSet. """ if inplace: results = [ins for ins in self._inner_iter() if not func(ins)] for name, old_field in self.field_arrays.items(): self.field_arrays[name].content = [ins[name] for ins in results] + return self else: results = [ins for ins in self if not func(ins)] data = DataSet(results) for field_name, field in self.field_arrays.items(): data.field_arrays[field_name].to(field) + return data - def split(self, dev_ratio): - """Split the dataset into training and development(validation) set. + def split(self, ratio): + """将DataSet按照ratio的比例拆分，返回两个DataSet - :param float dev_ratio: the ratio of test set in all data. - :return (train_set, dev_set): - train_set: the training set - dev_set: the development set + :param ratio: float, 0', '']: continue to_tag, to_label = split_tag_label(to_label) - if is_transition_allowed(encoding_type, from_tag, from_label, to_tag, to_label): + if _is_transition_allowed(encoding_type, from_tag, from_label, to_tag, to_label): allowed_trans.append((from_id, to_id)) return allowed_trans -def is_transition_allowed(encoding_type, from_tag, from_label, to_tag, to_label): +def _is_transition_allowed(encoding_type, from_tag, from_label, to_tag, to_label): """ :param encoding_type: str, 支持"BIO", "BMES", "BEMSO"。 @@ -140,20 +140,22 @@ def is_transition_allowed(encoding_type, from_tag, from_label, to_tag, to_label) raise ValueError("Unexpect tag type {}. Expect only 'B', 'M', 'E', 'S', 'O'.".format(from_tag)) else: - raise ValueError("Only support BIO, BMES encoding type, got {}.".format(encoding_type)) + raise ValueError("Only support BIO, BMES, BMESO encoding type, got {}.".format(encoding_type)) class ConditionalRandomField(nn.Module): - """ - - :param int num_tags: 标签的数量。 - :param bool include_start_end_trans: 是否包含起始tag - :param list allowed_transitions: ``List[Tuple[from_tag_id(int), to_tag_id(int)]]``. 允许的跃迁，可以通过allowed_transitions()得到。 - 如果为None，则所有跃迁均为合法 - :param str initial_method: - """ - - def __init__(self, num_tags, include_start_end_trans=False, allowed_transitions=None, initial_method=None): + def __init__(self, num_tags, include_start_end_trans=False, allowed_transitions=None, + initial_method=None): + """条件随机场。 + 提供forward()以及viterbi_decode()两个方法，分别用于训练与inference。 + + :param num_tags: int, 标签的数量 + :param include_start_end_trans: bool, 是否考虑各个tag作为开始以及结尾的分数。 + :param allowed_transitions: List[Tuple[from_tag_id(int), to_tag_id(int)]], 内部的Tuple[from_tag_id(int), + to_tag_id(int)]视为允许发生的跃迁，其他没有包含的跃迁认为是禁止跃迁，可以通过 + allowed_transitions()函数得到；如果为None，则所有跃迁均为合法 + :param initial_method: str, 初始化方法。见initial_parameter + """ super(ConditionalRandomField, self).__init__() self.include_start_end_trans = include_start_end_trans @@ -168,18 +170,12 @@ class ConditionalRandomField(nn.Module): if allowed_transitions is None: constrain = torch.zeros(num_tags + 2, num_tags + 2) else: - constrain = torch.ones(num_tags + 2, num_tags + 2) * -1000 + constrain = torch.new_full((num_tags+2, num_tags+2), fill_value=-10000.0, dtype=torch.float) for from_tag_id, to_tag_id in allowed_transitions: constrain[from_tag_id, to_tag_id] = 0 self._constrain = nn.Parameter(constrain, requires_grad=False) - # self.reset_parameter() initial_parameter(self, initial_method) - def reset_parameter(self): - nn.init.xavier_normal_(self.trans_m) - if self.include_start_end_trans: - nn.init.normal_(self.start_scores) - nn.init.normal_(self.end_scores) def _normalizer_likelihood(self, logits, mask): """Computes the (batch_size,) denominator term for the log-likelihood, which is the @@ -239,10 +235,11 @@ class ConditionalRandomField(nn.Module): def forward(self, feats, tags, mask): """ - Calculate the neg log likelihood - :param feats:FloatTensor, batch_size x max_len x num_tags - :param tags:LongTensor, batch_size x max_len - :param mask:ByteTensor batch_size x max_len + 用于计算CRF的前向loss，返回值为一个batch_size的FloatTensor，可能需要mean()求得loss。 + + :param feats:FloatTensor, batch_size x max_len x num_tags，特征矩阵。 + :param tags:LongTensor, batch_size x max_len，标签矩阵。 + :param mask:ByteTensor batch_size x max_len，为0的位置认为是padding。 :return:FloatTensor, batch_size """ feats = feats.transpose(0, 1) @@ -253,28 +250,27 @@ class ConditionalRandomField(nn.Module): return all_path_score - gold_path_score - def viterbi_decode(self, data, mask, get_score=False, unpad=False): - """Given a feats matrix, return best decode path and best score. + def viterbi_decode(self, feats, mask, unpad=False): + """给定一个特征矩阵以及转移分数矩阵，计算出最佳的路径以及对应的分数 - :param data:FloatTensor, batch_size x max_len x num_tags - :param mask:ByteTensor batch_size x max_len - :param get_score: bool, whether to output the decode score. - :param unpad: bool, 是否将结果unpad, - 如果False, 返回的是batch_size x max_len的tensor， - 如果True，返回的是List[List[int]], List[int]为每个sequence的label，已经unpadding了，即每个 - List[int]的长度是这个sample的有效长度 - :return: 如果get_score为False，返回结果根据unpadding变动 - 如果get_score为True, 返回 (paths, List[float], )。第一个仍然是解码后的路径(根据unpad变化)，第二个List[Float] - 为每个seqence的解码分数。 + :param feats: FloatTensor, batch_size x max_len x num_tags，特征矩阵。 + :param mask: ByteTensor, batch_size x max_len, 为0的位置认为是pad；如果为None，则认为没有padding。 + :param unpad: bool, 是否将结果删去padding, + False, 返回的是batch_size x max_len的tensor， + True，返回的是List[List[int]], 内部的List[int]为每个sequence的label，已经除去pad部分，即每个List[int] + 的长度是这个sample的有效长度。 + :return: 返回 (paths, scores)。 + paths: 是解码后的路径, 其值参照unpad参数. + scores: torch.FloatTensor, size为(batch_size,), 对应每个最优路径的分数。 """ - batch_size, seq_len, n_tags = data.size() - data = data.transpose(0, 1).data # L, B, H + batch_size, seq_len, n_tags = feats.size() + feats = feats.transpose(0, 1).data # L, B, H mask = mask.transpose(0, 1).data.byte() # L, B # dp - vpath = data.new_zeros((seq_len, batch_size, n_tags), dtype=torch.long) - vscore = data[0] + vpath = feats.new_zeros((seq_len, batch_size, n_tags), dtype=torch.long) + vscore = feats[0] transitions = self._constrain.data.clone() transitions[:n_tags, :n_tags] += self.trans_m.data if self.include_start_end_trans: @@ -285,23 +281,24 @@ class ConditionalRandomField(nn.Module): trans_score = transitions[:n_tags, :n_tags].view(1, n_tags, n_tags).data for i in range(1, seq_len): prev_score = vscore.view(batch_size, n_tags, 1) - cur_score = data[i].view(batch_size, 1, n_tags) + cur_score = feats[i].view(batch_size, 1, n_tags) score = prev_score + trans_score + cur_score best_score, best_dst = score.max(1) vpath[i] = best_dst vscore = best_score.masked_fill(mask[i].eq(0).view(batch_size, 1), 0) + \ vscore.masked_fill(mask[i].view(batch_size, 1), 0) - vscore += transitions[:n_tags, n_tags+1].view(1, -1) + if self.include_start_end_trans: + vscore += transitions[:n_tags, n_tags+1].view(1, -1) # backtrace - batch_idx = torch.arange(batch_size, dtype=torch.long, device=data.device) - seq_idx = torch.arange(seq_len, dtype=torch.long, device=data.device) + batch_idx = torch.arange(batch_size, dtype=torch.long, device=feats.device) + seq_idx = torch.arange(seq_len, dtype=torch.long, device=feats.device) lens = (mask.long().sum(0) - 1) # idxes [L, B], batched idx from seq_len-1 to 0 idxes = (lens.view(1,-1) - seq_idx.view(-1,1)) % seq_len - ans = data.new_empty((seq_len, batch_size), dtype=torch.long) + ans = feats.new_empty((seq_len, batch_size), dtype=torch.long) ans_score, last_tags = vscore.max(1) ans[idxes[0], batch_idx] = last_tags for i in range(seq_len - 1): diff --git a/fastNLP/modules/decoder/utils.py b/fastNLP/modules/decoder/utils.py new file mode 100644 index 00000000..6e35af9a --- /dev/null +++ b/fastNLP/modules/decoder/utils.py @@ -0,0 +1,70 @@ + +import torch + + +def log_sum_exp(x, dim=-1): + max_value, _ = x.max(dim=dim, keepdim=True) + res = torch.log(torch.sum(torch.exp(x - max_value), dim=dim, keepdim=True)) + max_value + return res.squeeze(dim) + + +def viterbi_decode(feats, transitions, mask=None, unpad=False): + """给定一个特征矩阵以及转移分数矩阵，计算出最佳的路径以及对应的分数 + + :param feats: FloatTensor, batch_size x max_len x num_tags，特征矩阵。 + :param transitions: FloatTensor, n_tags x n_tags。[i, j]位置的值认为是从tag i到tag j的转换。 + :param mask: ByteTensor, batch_size x max_len, 为0的位置认为是pad；如果为None，则认为没有padding。 + :param unpad: bool, 是否将结果删去padding, + False, 返回的是batch_size x max_len的tensor， + True，返回的是List[List[int]], 内部的List[int]为每个sequence的label，已经除去pad部分，即每个List[int]的长度是 + 这个sample的有效长度。 + :return: 返回 (paths, scores)。 + paths: 是解码后的路径, 其值参照unpad参数. + scores: torch.FloatTensor, size为(batch_size,), 对应每个最优路径的分数。 + + """ + batch_size, seq_len, n_tags = feats.size() + assert n_tags==transitions.size(0) and n_tags==transitions.size(1), "The shapes of transitions and feats are not " \ + "compatible." + feats = feats.transpose(0, 1).data # L, B, H + if mask is not None: + mask = mask.transpose(0, 1).data.byte() # L, B + else: + mask = feats.new_ones((seq_len, batch_size), dtype=torch.uint8) + + # dp + vpath = feats.new_zeros((seq_len, batch_size, n_tags), dtype=torch.long) + vscore = feats[0] + + vscore += transitions[n_tags, :n_tags] + trans_score = transitions[:n_tags, :n_tags].view(1, n_tags, n_tags).data + for i in range(1, seq_len): + prev_score = vscore.view(batch_size, n_tags, 1) + cur_score = feats[i].view(batch_size, 1, n_tags) + score = prev_score + trans_score + cur_score + best_score, best_dst = score.max(1) + vpath[i] = best_dst + vscore = best_score.masked_fill(mask[i].eq(0).view(batch_size, 1), 0) + \ + vscore.masked_fill(mask[i].view(batch_size, 1), 0) + + # backtrace + batch_idx = torch.arange(batch_size, dtype=torch.long, device=feats.device) + seq_idx = torch.arange(seq_len, dtype=torch.long, device=feats.device) + lens = (mask.long().sum(0) - 1) + # idxes [L, B], batched idx from seq_len-1 to 0 + idxes = (lens.view(1, -1) - seq_idx.view(-1, 1)) % seq_len + + ans = feats.new_empty((seq_len, batch_size), dtype=torch.long) + ans_score, last_tags = vscore.max(1) + ans[idxes[0], batch_idx] = last_tags + for i in range(seq_len - 1): + last_tags = vpath[idxes[i], batch_idx, last_tags] + ans[idxes[i + 1], batch_idx] = last_tags + ans = ans.transpose(0, 1) + if unpad: + paths = [] + for idx, seq_len in enumerate(lens): + paths.append(ans[idx, :seq_len + 1].tolist()) + else: + paths = ans + return paths, ans_score \ No newline at end of file diff --git a/reproduction/Chinese_word_segmentation/models/cws_model.py b/reproduction/Chinese_word_segmentation/models/cws_model.py index daefc380..13632207 100644 --- a/reproduction/Chinese_word_segmentation/models/cws_model.py +++ b/reproduction/Chinese_word_segmentation/models/cws_model.py @@ -183,7 +183,7 @@ class CWSBiLSTMCRF(BaseModel): masks = seq_lens_to_mask(seq_lens) feats = self.encoder_model(chars, bigrams, seq_lens) feats = self.decoder_model(feats) - probs = self.crf.viterbi_decode(feats, masks, get_score=False) + paths, _ = self.crf.viterbi_decode(feats, masks) - return {'pred': probs, 'seq_lens':seq_lens} + return {'pred': paths, 'seq_lens':seq_lens} diff --git a/reproduction/Chinese_word_segmentation/models/cws_transformer.py b/reproduction/Chinese_word_segmentation/models/cws_transformer.py index 736edade..d49ce3a9 100644 --- a/reproduction/Chinese_word_segmentation/models/cws_transformer.py +++ b/reproduction/Chinese_word_segmentation/models/cws_transformer.py @@ -72,9 +72,9 @@ class TransformerCWS(nn.Module): feats = self.transformer(x, masks) feats = self.fc2(feats) - probs = self.crf.viterbi_decode(feats, masks, get_score=False) + paths, _ = self.crf.viterbi_decode(feats, masks) - return {'pred': probs, 'seq_lens':seq_lens} + return {'pred': paths, 'seq_lens':seq_lens} class NoamOpt(torch.optim.Optimizer): From 15cdee827a3a6788e4b16127b000c5cd60c72047 Mon Sep 17 00:00:00 2001 From: ChenXin Date: Mon, 22 Apr 2019 11:34:45 +0800 Subject: [PATCH 5/9] =?UTF-8?q?=E6=A0=B7=E4=BE=8B=E7=89=88=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/Makefile | 3 + fastNLP/api/__init__.py | 3 - fastNLP/api/api.py | 83 ++++++++++++++++++------- fastNLP/modules/aggregator/attention.py | 6 +- 4 files changed, 70 insertions(+), 25 deletions(-) diff --git a/docs/Makefile b/docs/Makefile index 6a5c7375..6f2f2821 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -16,6 +16,9 @@ help: apidoc: @$(SPHINXAPIDOC) -f -o source ../fastNLP +server: + cd build/html && python -m http.server + .PHONY: help Makefile # Catch-all target: route all unknown targets to Sphinx using the new diff --git a/fastNLP/api/__init__.py b/fastNLP/api/__init__.py index ae31b80b..a21a4c42 100644 --- a/fastNLP/api/__init__.py +++ b/fastNLP/api/__init__.py @@ -1,4 +1 @@ -""" - 这是 API 部分的注释 -""" from .api import CWS, POS, Parser diff --git a/fastNLP/api/api.py b/fastNLP/api/api.py index b001629c..f088b121 100644 --- a/fastNLP/api/api.py +++ b/fastNLP/api/api.py @@ -1,5 +1,39 @@ """ -API.API 的文档 +api.api的介绍文档 + 直接缩进会把上面的文字变成标题 + +空行缩进的写法比较合理 + + 比较合理 + +*这里是斜体内容* + +**这里是粗体内容** + +数学公式块 + +.. math:: + E = mc^2 + +.. note:: + 注解型提示。 + +.. warning:: + 警告型提示。 + +.. seealso:: + `参考与超链接 `_ + +普通代码块需要空一行, Example:: + + from fitlog import fitlog + fitlog.commit() + +普通下标和上标: + +H\ :sub:`2`\ O + +E = mc\ :sup:`2` """ import warnings @@ -28,6 +62,9 @@ model_urls = { class API: + """ + 这是 API 类的文档 + """ def __init__(self): self.pipeline = None self._dict = None @@ -73,8 +110,9 @@ class POS(API): self.load(model_path, device) def predict(self, content): - """ - + """predict函数的介绍， + 函数介绍的第二句，这句话不会换行 + :param content: list of list of str. Each string is a token(word). :return answer: list of list of str. Each string is a tag. """ @@ -140,13 +178,14 @@ class POS(API): class CWS(API): - def __init__(self, model_path=None, device='cpu'): - """ - 中文分词高级接口。 + """ + 中文分词高级接口。 - :param model_path: 当model_path为None，使用默认位置的model。如果默认位置不存在，则自动下载模型 - :param device: str，可以为'cpu', 'cuda'或'cuda:0'等。会将模型load到相应device进行推断。 - """ + :param model_path: 当model_path为None，使用默认位置的model。如果默认位置不存在，则自动下载模型 + :param device: str，可以为'cpu', 'cuda'或'cuda:0'等。会将模型load到相应device进行推断。 + """ + def __init__(self, model_path=None, device='cpu'): + super(CWS, self).__init__() if model_path is None: model_path = model_urls['cws'] @@ -187,18 +226,20 @@ class CWS(API): def test(self, filepath): """ 传入一个分词文件路径，返回该数据集上分词f1, precision, recall。 - 分词文件应该为: - 1 编者按 编者按 NN O 11 nmod:topic - 2 ： ： PU O 11 punct - 3 7月 7月 NT DATE 4 compound:nn - 4 12日 12日 NT DATE 11 nmod:tmod - 5 ， ， PU O 11 punct - - 1 这 这 DT O 3 det - 2 款 款 M O 1 mark:clf - 3 飞行 飞行 NN O 8 nsubj - 4 从 从 P O 5 case - 5 外型 外型 NN O 8 nmod:prep + 分词文件应该为:: + + 1 编者按 编者按 NN O 11 nmod:topic + 2 ： ： PU O 11 punct + 3 7月 7月 NT DATE 4 compound:nn + 4 12日 12日 NT DATE 11 nmod:tmod + 5 ， ， PU O 11 punct + + 1 这 这 DT O 3 det + 2 款 款 M O 1 mark:clf + 3 飞行 飞行 NN O 8 nsubj + 4 从 从 P O 5 case + 5 外型 外型 NN O 8 nmod:prep + 以空行分割两个句子，有内容的每行有7列。 :param filepath: str, 文件路径路径。 diff --git a/fastNLP/modules/aggregator/attention.py b/fastNLP/modules/aggregator/attention.py index 33d73a07..4155fdd6 100644 --- a/fastNLP/modules/aggregator/attention.py +++ b/fastNLP/modules/aggregator/attention.py @@ -112,12 +112,15 @@ class MultiHeadAttention(nn.Module): class BiAttention(nn.Module): """Bi Attention module Calculate Bi Attention matrix `e` + .. math:: + \begin{array}{ll} \\ e_ij = {a}^{\mathbf{T}}_{i}{b}_{j} \\ a_i = b_j = \end{array} + """ def __init__(self): @@ -131,7 +134,8 @@ class BiAttention(nn.Module): :param torch.Tensor x1_len: [batch_size, x1_seq_len] 第一句的0/1mask矩阵 :param torch.Tensor x2_len: [batch_size, x2_seq_len] 第二句的0/1mask矩阵 :return: torch.Tensor out_x1: [batch_size, x1_seq_len, hidden_size] 第一句attend到的特征表示 - torch.Tensor out_x2: [batch_size, x2_seq_len, hidden_size] 第一句attend到的特征表示 + torch.Tensor out_x2: [batch_size, x2_seq_len, hidden_size] 第一句attend到的特征表示 + """ assert in_x1.size()[0] == in_x2.size()[0] From c344f7a2f9f637d0c5d6b2b059d59a69d7fb885f Mon Sep 17 00:00:00 2001 From: yunfan Date: Mon, 22 Apr 2019 01:04:10 +0800 Subject: [PATCH 6/9] - add pad sequence for lstm - add csv, conll, json filereader - update dataloader - remove useless dataloader - fix trainer loss print - fix tests --- fastNLP/api/api.py | 81 ++- fastNLP/core/dataset.py | 6 +- fastNLP/core/trainer.py | 3 +- fastNLP/io/dataset_loader.py | 700 +++----------------------- fastNLP/io/file_reader.py | 112 +++++ fastNLP/modules/encoder/lstm.py | 39 +- test/core/test_dataset.py | 24 +- test/data_for_tests/sample_snli.jsonl | 3 + test/io/test_dataset_loader.py | 19 +- 9 files changed, 316 insertions(+), 671 deletions(-) create mode 100644 fastNLP/io/file_reader.py create mode 100644 test/data_for_tests/sample_snli.jsonl diff --git a/fastNLP/api/api.py b/fastNLP/api/api.py index 53a80131..512f485b 100644 --- a/fastNLP/api/api.py +++ b/fastNLP/api/api.py @@ -9,7 +9,7 @@ from fastNLP.core.dataset import DataSet from fastNLP.api.utils import load_url from fastNLP.api.processor import ModelProcessor -from fastNLP.io.dataset_loader import ConllCWSReader, ConllxDataLoader +from fastNLP.io.dataset_loader import cut_long_sentence, ConllLoader from fastNLP.core.instance import Instance from fastNLP.api.pipeline import Pipeline from fastNLP.core.metrics import SpanFPreRecMetric @@ -23,6 +23,85 @@ model_urls = { } +class ConllCWSReader(object): + """Deprecated. Use ConllLoader for all types of conll-format files.""" + def __init__(self): + pass + + def load(self, path, cut_long_sent=False): + """ + 返回的DataSet只包含raw_sentence这个field，内容为str。 + 假定了输入为conll的格式，以空行隔开两个句子，每行共7列，即 + :: + + 1 编者按 编者按 NN O 11 nmod:topic + 2 ： ： PU O 11 punct + 3 7月 7月 NT DATE 4 compound:nn + 4 12日 12日 NT DATE 11 nmod:tmod + 5 ， ， PU O 11 punct + + 1 这 这 DT O 3 det + 2 款 款 M O 1 mark:clf + 3 飞行 飞行 NN O 8 nsubj + 4 从 从 P O 5 case + 5 外型 外型 NN O 8 nmod:prep + + """ + datalist = [] + with open(path, 'r', encoding='utf-8') as f: + sample = [] + for line in f: + if line.startswith('\n'): + datalist.append(sample) + sample = [] + elif line.startswith('#'): + continue + else: + sample.append(line.strip().split()) + if len(sample) > 0: + datalist.append(sample) + + ds = DataSet() + for sample in datalist: + # print(sample) + res = self.get_char_lst(sample) + if res is None: + continue + line = ' '.join(res) + if cut_long_sent: + sents = cut_long_sentence(line) + else: + sents = [line] + for raw_sentence in sents: + ds.append(Instance(raw_sentence=raw_sentence)) + return ds + + def get_char_lst(self, sample): + if len(sample) == 0: + return None + text = [] + for w in sample: + t1, t2, t3, t4 = w[1], w[3], w[6], w[7] + if t3 == '_': + return None + text.append(t1) + return text + +class ConllxDataLoader(ConllLoader): + """返回“词级别”的标签信息，包括词、词性、（句法）头依赖、（句法）边标签。跟``ZhConllPOSReader``完全不同。 + + Deprecated. Use ConllLoader for all types of conll-format files. + """ + def __init__(self): + headers = [ + 'words', 'pos_tags', 'heads', 'labels', + ] + indexs = [ + 1, 3, 6, 7, + ] + super(ConllxDataLoader, self).__init__(headers=headers, indexs=indexs) + + class API: def __init__(self): self.pipeline = None diff --git a/fastNLP/core/dataset.py b/fastNLP/core/dataset.py index 24376a72..3ef61177 100644 --- a/fastNLP/core/dataset.py +++ b/fastNLP/core/dataset.py @@ -373,6 +373,9 @@ class DataSet(object): :return dataset: the read data set """ + import warnings + warnings.warn('read_csv is deprecated, use CSVLoader instead', + category=DeprecationWarning) with open(csv_path, "r") as f: start_idx = 0 if headers is None: @@ -398,9 +401,6 @@ class DataSet(object): _dict[header].append(content) return cls(_dict) - # def read_pos(self): - # return DataLoaderRegister.get_reader('read_pos') - def save(self, path): """Save the DataSet object as pickle. diff --git a/fastNLP/core/trainer.py b/fastNLP/core/trainer.py index d9aa520f..1b5c1edf 100644 --- a/fastNLP/core/trainer.py +++ b/fastNLP/core/trainer.py @@ -268,8 +268,9 @@ class Trainer(object): self.callback_manager.on_step_end() if self.step % self.print_every == 0: + avg_loss = float(avg_loss) / self.print_every if self.use_tqdm: - print_output = "loss:{0:<6.5f}".format(avg_loss / self.print_every) + print_output = "loss:{0:<6.5f}".format(avg_loss) pbar.update(self.print_every) else: end = time.time() diff --git a/fastNLP/io/dataset_loader.py b/fastNLP/io/dataset_loader.py index e33384a8..5657e194 100644 --- a/fastNLP/io/dataset_loader.py +++ b/fastNLP/io/dataset_loader.py @@ -1,71 +1,13 @@ import os import json +from nltk.tree import Tree from fastNLP.core.dataset import DataSet from fastNLP.core.instance import Instance -from fastNLP.io.base_loader import DataLoaderRegister +from fastNLP.io.file_reader import read_csv, read_json, read_conll -def convert_seq_dataset(data): - """Create an DataSet instance that contains no labels. - - :param data: list of list of strings, [num_examples, *]. - Example:: - - [ - [word_11, word_12, ...], - ... - ] - - :return: a DataSet. - """ - dataset = DataSet() - for word_seq in data: - dataset.append(Instance(word_seq=word_seq)) - return dataset - - -def convert_seq2tag_dataset(data): - """Convert list of data into DataSet. - - :param data: list of list of strings, [num_examples, *]. - Example:: - - [ - [ [word_11, word_12, ...], label_1 ], - [ [word_21, word_22, ...], label_2 ], - ... - ] - - :return: a DataSet. - """ - dataset = DataSet() - for sample in data: - dataset.append(Instance(word_seq=sample[0], label=sample[1])) - return dataset - - -def convert_seq2seq_dataset(data): - """Convert list of data into DataSet. - - :param data: list of list of strings, [num_examples, *]. - Example:: - - [ - [ [word_11, word_12, ...], [label_1, label_1, ...] ], - [ [word_21, word_22, ...], [label_2, label_1, ...] ], - ... - ] - - :return: a DataSet. - """ - dataset = DataSet() - for sample in data: - dataset.append(Instance(word_seq=sample[0], label_seq=sample[1])) - return dataset - - -def download_from_url(url, path): +def _download_from_url(url, path): from tqdm import tqdm import requests @@ -81,7 +23,7 @@ def download_from_url(url, path): t.update(len(chunk)) return -def uncompress(src, dst): +def _uncompress(src, dst): import zipfile, gzip, tarfile, os def unzip(src, dst): @@ -134,241 +76,6 @@ class DataSetLoader: raise NotImplementedError -class NativeDataSetLoader(DataSetLoader): - """A simple example of DataSetLoader - - """ - - def __init__(self): - super(NativeDataSetLoader, self).__init__() - - def load(self, path): - ds = DataSet.read_csv(path, headers=("raw_sentence", "label"), sep="\t") - ds.set_input("raw_sentence") - ds.set_target("label") - return ds - - -DataLoaderRegister.set_reader(NativeDataSetLoader, 'read_naive') - - -class RawDataSetLoader(DataSetLoader): - """A simple example of raw data reader - - """ - - def __init__(self): - super(RawDataSetLoader, self).__init__() - - def load(self, data_path, split=None): - with open(data_path, "r", encoding="utf-8") as f: - lines = f.readlines() - lines = lines if split is None else [l.split(split) for l in lines] - lines = list(filter(lambda x: len(x) > 0, lines)) - return self.convert(lines) - - def convert(self, data): - return convert_seq_dataset(data) - - -DataLoaderRegister.set_reader(RawDataSetLoader, 'read_rawdata') - - -class DummyPOSReader(DataSetLoader): - """A simple reader for a dummy POS tagging dataset. - - In these datasets, each line are divided by "\t". The first Col is the vocabulary and the second - Col is the label. Different sentence are divided by an empty line. - E.g:: - - Tom label1 - and label2 - Jerry label1 - . label3 - (separated by an empty line) - Hello label4 - world label5 - ! label3 - - In this example, there are two sentences "Tom and Jerry ." and "Hello world !". Each word has its own label. - """ - - def __init__(self): - super(DummyPOSReader, self).__init__() - - def load(self, data_path): - """ - :return data: three-level list - Example:: - [ - [ [word_11, word_12, ...], [label_1, label_1, ...] ], - [ [word_21, word_22, ...], [label_2, label_1, ...] ], - ... - ] - """ - with open(data_path, "r", encoding="utf-8") as f: - lines = f.readlines() - data = self.parse(lines) - return self.convert(data) - - @staticmethod - def parse(lines): - data = [] - sentence = [] - for line in lines: - line = line.strip() - if len(line) > 1: - sentence.append(line.split('\t')) - else: - words = [] - labels = [] - for tokens in sentence: - words.append(tokens[0]) - labels.append(tokens[1]) - data.append([words, labels]) - sentence = [] - if len(sentence) != 0: - words = [] - labels = [] - for tokens in sentence: - words.append(tokens[0]) - labels.append(tokens[1]) - data.append([words, labels]) - return data - - def convert(self, data): - """Convert lists of strings into Instances with Fields. - """ - return convert_seq2seq_dataset(data) - - -DataLoaderRegister.set_reader(DummyPOSReader, 'read_pos') - - -class DummyCWSReader(DataSetLoader): - """Load pku dataset for Chinese word segmentation. - """ - def __init__(self): - super(DummyCWSReader, self).__init__() - - def load(self, data_path, max_seq_len=32): - """Load pku dataset for Chinese word segmentation. - CWS (Chinese Word Segmentation) pku training dataset format: - 1. Each line is a sentence. - 2. Each word in a sentence is separated by space. - This function convert the pku dataset into three-level lists with labels . - B: beginning of a word - M: middle of a word - E: ending of a word - S: single character - - :param str data_path: path to the data set. - :param max_seq_len: int, the maximum length of a sequence. If a sequence is longer than it, split it into - several sequences. - :return: three-level lists - """ - assert isinstance(max_seq_len, int) and max_seq_len > 0 - with open(data_path, "r", encoding="utf-8") as f: - sentences = f.readlines() - data = [] - for sent in sentences: - tokens = sent.strip().split() - words = [] - labels = [] - for token in tokens: - if len(token) == 1: - words.append(token) - labels.append("S") - else: - words.append(token[0]) - labels.append("B") - for idx in range(1, len(token) - 1): - words.append(token[idx]) - labels.append("M") - words.append(token[-1]) - labels.append("E") - num_samples = len(words) // max_seq_len - if len(words) % max_seq_len != 0: - num_samples += 1 - for sample_idx in range(num_samples): - start = sample_idx * max_seq_len - end = (sample_idx + 1) * max_seq_len - seq_words = words[start:end] - seq_labels = labels[start:end] - data.append([seq_words, seq_labels]) - return self.convert(data) - - def convert(self, data): - return convert_seq2seq_dataset(data) - - -class DummyClassificationReader(DataSetLoader): - """Loader for a dummy classification data set""" - - def __init__(self): - super(DummyClassificationReader, self).__init__() - - def load(self, data_path): - assert os.path.exists(data_path) - with open(data_path, "r", encoding="utf-8") as f: - lines = f.readlines() - data = self.parse(lines) - return self.convert(data) - - @staticmethod - def parse(lines): - """每行第一个token是标签，其余是字/词；由空格分隔。 - - :param lines: lines from dataset - :return: list(list(list())): the three level of lists are words, sentence, and dataset - """ - dataset = list() - for line in lines: - line = line.strip().split() - label = line[0] - words = line[1:] - if len(words) <= 1: - continue - - sentence = [words, label] - dataset.append(sentence) - return dataset - - def convert(self, data): - return convert_seq2tag_dataset(data) - - -class DummyLMReader(DataSetLoader): - """A Dummy Language Model Dataset Reader - """ - def __init__(self): - super(DummyLMReader, self).__init__() - - def load(self, data_path): - if not os.path.exists(data_path): - raise FileNotFoundError("file {} not found.".format(data_path)) - with open(data_path, "r", encoding="utf=8") as f: - text = " ".join(f.readlines()) - tokens = text.strip().split() - data = self.sentence_cut(tokens) - return self.convert(data) - - def sentence_cut(self, tokens, sentence_length=15): - start_idx = 0 - data_set = [] - for idx in range(len(tokens) // sentence_length): - x = tokens[start_idx * idx: start_idx * idx + sentence_length] - y = tokens[start_idx * idx + 1: start_idx * idx + sentence_length + 1] - if start_idx * idx + sentence_length + 1 >= len(tokens): - # ad hoc - y.extend([""]) - data_set.append([x, y]) - return data_set - - def convert(self, data): - pass - - class PeopleDailyCorpusLoader(DataSetLoader): """人民日报数据集 """ @@ -448,8 +155,9 @@ class PeopleDailyCorpusLoader(DataSetLoader): class ConllLoader: - def __init__(self, headers, indexs=None): + def __init__(self, headers, indexs=None, dropna=True): self.headers = headers + self.dropna = dropna if indexs is None: self.indexs = list(range(len(self.headers))) else: @@ -458,33 +166,10 @@ class ConllLoader: self.indexs = indexs def load(self, path): - datalist = [] - with open(path, 'r', encoding='utf-8') as f: - sample = [] - start = next(f) - if '-DOCSTART-' not in start: - sample.append(start.split()) - for line in f: - if line.startswith('\n'): - if len(sample): - datalist.append(sample) - sample = [] - elif line.startswith('#'): - continue - else: - sample.append(line.split()) - if len(sample) > 0: - datalist.append(sample) - - data = [self.get_one(sample) for sample in datalist] - data = filter(lambda x: x is not None, data) - ds = DataSet() - for sample in data: - ins = Instance() - for name, idx in zip(self.headers, self.indexs): - ins.add_field(field_name=name, field=sample[idx]) - ds.append(ins) + for idx, data in read_conll(path, indexes=self.indexs, dropna=self.dropna): + ins = {h:data[idx] for h, idx in zip(self.headers, self.indexs)} + ds.append(Instance(**ins)) return ds def get_one(self, sample): @@ -499,9 +184,7 @@ class Conll2003Loader(ConllLoader): """Loader for conll2003 dataset More information about the given dataset cound be found on - https://sites.google.com/site/ermasoftware/getting-started/ne-tagging-conll2003-data - - Deprecated. Use ConllLoader for all types of conll-format files. + https://sites.google.com/site/ermasoftware/getting-started/ne-tagging-conll2003-data """ def __init__(self): headers = [ @@ -510,194 +193,6 @@ class Conll2003Loader(ConllLoader): super(Conll2003Loader, self).__init__(headers=headers) -class SNLIDataSetReader(DataSetLoader): - """A data set loader for SNLI data set. - - """ - def __init__(self): - super(SNLIDataSetReader, self).__init__() - - def load(self, path_list): - """ - - :param list path_list: A list of file name, in the order of premise file, hypothesis file, and label file. - :return: A DataSet object. - """ - assert len(path_list) == 3 - line_set = [] - for file in path_list: - if not os.path.exists(file): - raise FileNotFoundError("file {} NOT found".format(file)) - - with open(file, 'r', encoding='utf-8') as f: - lines = f.readlines() - line_set.append(lines) - - premise_lines, hypothesis_lines, label_lines = line_set - assert len(premise_lines) == len(hypothesis_lines) and len(premise_lines) == len(label_lines) - - data_set = [] - for premise, hypothesis, label in zip(premise_lines, hypothesis_lines, label_lines): - p = premise.strip().split() - h = hypothesis.strip().split() - l = label.strip() - data_set.append([p, h, l]) - - return self.convert(data_set) - - def convert(self, data): - """Convert a 3D list to a DataSet object. - - :param data: A 3D tensor. - Example:: - [ - [ [premise_word_11, premise_word_12, ...], [hypothesis_word_11, hypothesis_word_12, ...], [label_1] ], - [ [premise_word_21, premise_word_22, ...], [hypothesis_word_21, hypothesis_word_22, ...], [label_2] ], - ... - ] - - :return: A DataSet object. - """ - - data_set = DataSet() - - for example in data: - p, h, l = example - # list, list, str - instance = Instance() - instance.add_field("premise", p) - instance.add_field("hypothesis", h) - instance.add_field("truth", l) - data_set.append(instance) - data_set.apply(lambda ins: len(ins["premise"]), new_field_name="premise_len") - data_set.apply(lambda ins: len(ins["hypothesis"]), new_field_name="hypothesis_len") - data_set.set_input("premise", "hypothesis", "premise_len", "hypothesis_len") - data_set.set_target("truth") - return data_set - - -class ConllCWSReader(object): - """Deprecated. Use ConllLoader for all types of conll-format files.""" - def __init__(self): - pass - - def load(self, path, cut_long_sent=False): - """ - 返回的DataSet只包含raw_sentence这个field，内容为str。 - 假定了输入为conll的格式，以空行隔开两个句子，每行共7列，即 - :: - - 1 编者按 编者按 NN O 11 nmod:topic - 2 ： ： PU O 11 punct - 3 7月 7月 NT DATE 4 compound:nn - 4 12日 12日 NT DATE 11 nmod:tmod - 5 ， ， PU O 11 punct - - 1 这 这 DT O 3 det - 2 款 款 M O 1 mark:clf - 3 飞行 飞行 NN O 8 nsubj - 4 从 从 P O 5 case - 5 外型 外型 NN O 8 nmod:prep - - """ - datalist = [] - with open(path, 'r', encoding='utf-8') as f: - sample = [] - for line in f: - if line.startswith('\n'): - datalist.append(sample) - sample = [] - elif line.startswith('#'): - continue - else: - sample.append(line.strip().split()) - if len(sample) > 0: - datalist.append(sample) - - ds = DataSet() - for sample in datalist: - # print(sample) - res = self.get_char_lst(sample) - if res is None: - continue - line = ' '.join(res) - if cut_long_sent: - sents = cut_long_sentence(line) - else: - sents = [line] - for raw_sentence in sents: - ds.append(Instance(raw_sentence=raw_sentence)) - return ds - - def get_char_lst(self, sample): - if len(sample) == 0: - return None - text = [] - for w in sample: - t1, t2, t3, t4 = w[1], w[3], w[6], w[7] - if t3 == '_': - return None - text.append(t1) - return text - - -class NaiveCWSReader(DataSetLoader): - """ - 这个reader假设了分词数据集为以下形式, 即已经用空格分割好内容了 - 例如:: - - 这是 fastNLP , 一个 非常 good 的 包 . - - 或者,即每个part后面还有一个pos tag - 例如:: - - 也/D 在/P 團員/Na 之中/Ng ，/COMMACATEGORY - - """ - - def __init__(self, in_word_splitter=None): - super(NaiveCWSReader, self).__init__() - self.in_word_splitter = in_word_splitter - - def load(self, filepath, in_word_splitter=None, cut_long_sent=False): - """ - 允许使用的情况有(默认以\t或空格作为seg) - 这是 fastNLP , 一个 非常 good 的 包 . - 和 - 也/D 在/P 團員/Na 之中/Ng ，/COMMACATEGORY - 如果splitter不为None则认为是第二种情况, 且我们会按splitter分割"也/D", 然后取第一部分. 例如"也/D".split('/')[0] - - :param filepath: - :param in_word_splitter: - :param cut_long_sent: - :return: - """ - if in_word_splitter == None: - in_word_splitter = self.in_word_splitter - dataset = DataSet() - with open(filepath, 'r') as f: - for line in f: - line = line.strip() - if len(line.replace(' ', '')) == 0: # 不能接受空行 - continue - - if not in_word_splitter is None: - words = [] - for part in line.split(): - word = part.split(in_word_splitter)[0] - words.append(word) - line = ' '.join(words) - if cut_long_sent: - sents = cut_long_sentence(line) - else: - sents = [line] - for sent in sents: - instance = Instance(raw_sentence=sent) - dataset.append(instance) - - return dataset - - def cut_long_sentence(sent, max_sample_length=200): """ 将长于max_sample_length的sentence截成多段，只会在有空格的地方发生截断。所以截取的句子可能长于或者短于max_sample_length @@ -727,103 +222,6 @@ def cut_long_sentence(sent, max_sample_length=200): return cutted_sentence -class ZhConllPOSReader(object): - """读取中文Conll格式。返回“字级别”的标签，使用BMES记号扩展原来的词级别标签。 - - Deprecated. Use ConllLoader for all types of conll-format files. - """ - def __init__(self): - pass - - def load(self, path): - """ - 返回的DataSet, 包含以下的field - words：list of str, - tag: list of str, 被加入了BMES tag, 比如原来的序列为['VP', 'NN', 'NN', ..]，会被认为是["S-VP", "B-NN", "M-NN",..] - 假定了输入为conll的格式，以空行隔开两个句子，每行共7列，即 - :: - - 1 编者按 编者按 NN O 11 nmod:topic - 2 ： ： PU O 11 punct - 3 7月 7月 NT DATE 4 compound:nn - 4 12日 12日 NT DATE 11 nmod:tmod - 5 ， ， PU O 11 punct - - 1 这 这 DT O 3 det - 2 款 款 M O 1 mark:clf - 3 飞行 飞行 NN O 8 nsubj - 4 从 从 P O 5 case - 5 外型 外型 NN O 8 nmod:prep - - """ - datalist = [] - with open(path, 'r', encoding='utf-8') as f: - sample = [] - for line in f: - if line.startswith('\n'): - datalist.append(sample) - sample = [] - elif line.startswith('#'): - continue - else: - sample.append(line.split('\t')) - if len(sample) > 0: - datalist.append(sample) - - ds = DataSet() - for sample in datalist: - # print(sample) - res = self.get_one(sample) - if res is None: - continue - char_seq = [] - pos_seq = [] - for word, tag in zip(res[0], res[1]): - char_seq.extend(list(word)) - if len(word) == 1: - pos_seq.append('S-{}'.format(tag)) - elif len(word) > 1: - pos_seq.append('B-{}'.format(tag)) - for _ in range(len(word) - 2): - pos_seq.append('M-{}'.format(tag)) - pos_seq.append('E-{}'.format(tag)) - else: - raise ValueError("Zero length of word detected.") - - ds.append(Instance(words=char_seq, - tag=pos_seq)) - - return ds - - def get_one(self, sample): - if len(sample) == 0: - return None - text = [] - pos_tags = [] - for w in sample: - t1, t2, t3, t4 = w[1], w[3], w[6], w[7] - if t3 == '_': - return None - text.append(t1) - pos_tags.append(t2) - return text, pos_tags - - -class ConllxDataLoader(ConllLoader): - """返回“词级别”的标签信息，包括词、词性、（句法）头依赖、（句法）边标签。跟``ZhConllPOSReader``完全不同。 - - Deprecated. Use ConllLoader for all types of conll-format files. - """ - def __init__(self): - headers = [ - 'words', 'pos_tags', 'heads', 'labels', - ] - indexs = [ - 1, 3, 6, 7, - ] - super(ConllxDataLoader, self).__init__(headers=headers, indexs=indexs) - - class SSTLoader(DataSetLoader): """load SST data in PTB tree format data source: https://nlp.stanford.edu/sentiment/trainDevTestTrees_PTB.zip @@ -842,10 +240,7 @@ class SSTLoader(DataSetLoader): """ :param path: str，存储数据的路径 - :return: DataSet。内含field有'words', 'pos_tags', 'heads', 'labels'(parser的label) - 类似于拥有以下结构, 一行为一个instance(sample) - words pos_tags heads labels - ['some', ..] ['NN', ...] [2, 3...] ['nn', 'nn'...] + :return: DataSet。 """ datalist = [] with open(path, 'r', encoding='utf-8') as f: @@ -860,7 +255,6 @@ class SSTLoader(DataSetLoader): @staticmethod def get_one(data, subtree): - from nltk.tree import Tree tree = Tree.fromstring(data) if subtree: return [(t.leaves(), t.label()) for t in tree.subtrees()] @@ -872,26 +266,72 @@ class JsonLoader(DataSetLoader): every line contains a json obj, like a dict fields is the dict key that need to be load """ - def __init__(self, **fields): + def __init__(self, dropna=False, fields=None): super(JsonLoader, self).__init__() - self.fields = {} - for k, v in fields.items(): - self.fields[k] = k if v is None else v + self.dropna = dropna + self.fields = None + self.fields_list = None + if fields: + self.fields = {} + for k, v in fields.items(): + self.fields[k] = k if v is None else v + self.fields_list = list(self.fields.keys()) + + def load(self, path): + ds = DataSet() + for idx, d in read_json(path, fields=self.fields_list, dropna=self.dropna): + ins = {self.fields[k]:v for k,v in d.items()} + ds.append(Instance(**ins)) + return ds + + +class SNLILoader(JsonLoader): + """ + data source: https://nlp.stanford.edu/projects/snli/snli_1.0.zip + """ + def __init__(self): + fields = { + 'sentence1_parse': 'words1', + 'sentence2_parse': 'words2', + 'gold_label': 'target', + } + super(SNLILoader, self).__init__(fields=fields) + + def load(self, path): + ds = super(SNLILoader, self).load(path) + def parse_tree(x): + t = Tree.fromstring(x) + return t.leaves() + ds.apply(lambda ins: parse_tree(ins['words1']), new_field_name='words1') + ds.apply(lambda ins: parse_tree(ins['words2']), new_field_name='words2') + ds.drop(lambda x: x['target'] == '-') + return ds + + +class CSVLoader(DataSetLoader): + """Load data from a CSV file and return a DataSet object. + + :param str csv_path: path to the CSV file + :param List[str] or Tuple[str] headers: headers of the CSV file + :param str sep: delimiter in CSV file. Default: "," + :param bool dropna: If True, drop rows that have less entries than headers. + :return dataset: the read data set + + """ + def __init__(self, headers=None, sep=",", dropna=True): + self.headers = headers + self.sep = sep + self.dropna = dropna def load(self, path): - with open(path, 'r', encoding='utf-8') as f: - datas = [json.loads(l) for l in f] ds = DataSet() - for d in datas: - ins = Instance() - for k, v in d.items(): - if k in self.fields: - ins.add_field(self.fields[k], v) - ds.append(ins) + for idx, data in read_csv(path, headers=self.headers, + sep=self.sep, dropna=self.dropna): + ds.append(Instance(**data)) return ds -def add_seg_tag(data): +def _add_seg_tag(data): """ :param data: list of ([word], [pos], [heads], [head_tags]) diff --git a/fastNLP/io/file_reader.py b/fastNLP/io/file_reader.py new file mode 100644 index 00000000..22766ebb --- /dev/null +++ b/fastNLP/io/file_reader.py @@ -0,0 +1,112 @@ +import json + + +def read_csv(path, encoding='utf-8', headers=None, sep=',', dropna=True): + """ + Construct a generator to read csv items + :param path: file path + :param encoding: file's encoding, default: utf-8 + :param headers: file's headers, if None, make file's first line as headers. default: None + :param sep: separator for each column. default: ',' + :param dropna: weather to ignore and drop invalid data, + if False, raise ValueError when reading invalid data. default: True + :return: generator, every time yield (line number, csv item) + """ + with open(path, 'r', encoding=encoding) as f: + start_idx = 0 + if headers is None: + headers = f.readline().rstrip('\r\n') + headers = headers.split(sep) + start_idx += 1 + elif not isinstance(headers, (list, tuple)): + raise TypeError("headers should be list or tuple, not {}." \ + .format(type(headers))) + for line_idx, line in enumerate(f, start_idx): + contents = line.rstrip('\r\n').split(sep) + if len(contents) != len(headers): + if dropna: + continue + else: + raise ValueError("Line {} has {} parts, while header has {} parts." \ + .format(line_idx, len(contents), len(headers))) + _dict = {} + for header, content in zip(headers, contents): + _dict[header] = content + yield line_idx, _dict + + +def read_json(path, encoding='utf-8', fields=None, dropna=True): + """ + Construct a generator to read json items + :param path: file path + :param encoding: file's encoding, default: utf-8 + :param fields: json object's fields that needed, if None, all fields are needed. default: None + :param dropna: weather to ignore and drop invalid data, + if False, raise ValueError when reading invalid data. default: True + :return: generator, every time yield (line number, json item) + """ + if fields: + fields = set(fields) + with open(path, 'r', encoding=encoding) as f: + for line_idx, line in enumerate(f): + data = json.loads(line) + if fields is None: + yield line_idx, data + continue + _res = {} + for k, v in data.items(): + if k in fields: + _res[k] = v + if len(_res) < len(fields): + if dropna: + continue + else: + raise ValueError('invalid instance at line: {}'.format(line_idx)) + yield line_idx, _res + + +def read_conll(path, encoding='utf-8', indexes=None, dropna=True): + """ + Construct a generator to read conll items + :param path: file path + :param encoding: file's encoding, default: utf-8 + :param indexes: conll object's column indexes that needed, if None, all columns are needed. default: None + :param dropna: weather to ignore and drop invalid data, + if False, raise ValueError when reading invalid data. default: True + :return: generator, every time yield (line number, conll item) + """ + def parse_conll(sample): + sample = list(map(list, zip(*sample))) + sample = [sample[i] for i in indexes] + for f in sample: + if len(f) <= 0: + raise ValueError('empty field') + return sample + with open(path, 'r', encoding=encoding) as f: + sample = [] + start = next(f) + if '-DOCSTART-' not in start: + sample.append(start.split()) + for line_idx, line in enumerate(f, 1): + if line.startswith('\n'): + if len(sample): + try: + res = parse_conll(sample) + sample = [] + yield line_idx, res + except Exception as e: + if dropna: + continue + raise ValueError('invalid instance at line: {}'.format(line_idx)) + elif line.startswith('#'): + continue + else: + sample.append(line.split()) + if len(sample) > 0: + try: + res = parse_conll(sample) + yield line_idx, res + except Exception as e: + if dropna: + return + raise ValueError('invalid instance at line: {}'.format(line_idx)) diff --git a/fastNLP/modules/encoder/lstm.py b/fastNLP/modules/encoder/lstm.py index 48c67a64..04f331f7 100644 --- a/fastNLP/modules/encoder/lstm.py +++ b/fastNLP/modules/encoder/lstm.py @@ -1,4 +1,6 @@ +import torch import torch.nn as nn +import torch.nn.utils.rnn as rnn from fastNLP.modules.utils import initial_parameter @@ -19,21 +21,44 @@ class LSTM(nn.Module): def __init__(self, input_size, hidden_size=100, num_layers=1, dropout=0.0, batch_first=True, bidirectional=False, bias=True, initial_method=None, get_hidden=False): super(LSTM, self).__init__() + self.batch_first = batch_first self.lstm = nn.LSTM(input_size, hidden_size, num_layers, bias=bias, batch_first=batch_first, dropout=dropout, bidirectional=bidirectional) self.get_hidden = get_hidden initial_parameter(self, initial_method) - def forward(self, x, h0=None, c0=None): + def forward(self, x, seq_lens=None, h0=None, c0=None): if h0 is not None and c0 is not None: - x, (ht, ct) = self.lstm(x, (h0, c0)) + hx = (h0, c0) else: - x, (ht, ct) = self.lstm(x) - if self.get_hidden: - return x, (ht, ct) + hx = None + if seq_lens is not None and not isinstance(x, rnn.PackedSequence): + print('padding') + sort_lens, sort_idx = torch.sort(seq_lens, dim=0, descending=True) + if self.batch_first: + x = x[sort_idx] + else: + x = x[:, sort_idx] + x = rnn.pack_padded_sequence(x, sort_lens, batch_first=self.batch_first) + output, hx = self.lstm(x, hx) # -> [N,L,C] + output, _ = rnn.pad_packed_sequence(output, batch_first=self.batch_first) + _, unsort_idx = torch.sort(sort_idx, dim=0, descending=False) + if self.batch_first: + output = output[unsort_idx] + else: + output = output[:, unsort_idx] else: - return x + output, hx = self.lstm(x, hx) + if self.get_hidden: + return output, hx + return output if __name__ == "__main__": - lstm = LSTM(10) + lstm = LSTM(input_size=2, hidden_size=2, get_hidden=False) + x = torch.randn((3, 5, 2)) + seq_lens = torch.tensor([5,1,2]) + y = lstm(x, seq_lens) + print(x) + print(y) + print(x.size(), y.size(), ) diff --git a/test/core/test_dataset.py b/test/core/test_dataset.py index 356b157a..4384a680 100644 --- a/test/core/test_dataset.py +++ b/test/core/test_dataset.py @@ -202,25 +202,11 @@ class TestDataSetMethods(unittest.TestCase): self.assertTrue(isinstance(ans, FieldArray)) self.assertEqual(ans.content, [[5, 6]] * 10) - def test_reader(self): - # 跑通即可 - ds = DataSet().read_naive("test/data_for_tests/tutorial_sample_dataset.csv") - self.assertTrue(isinstance(ds, DataSet)) - self.assertTrue(len(ds) > 0) - - ds = DataSet().read_rawdata("test/data_for_tests/people_daily_raw.txt") - self.assertTrue(isinstance(ds, DataSet)) - self.assertTrue(len(ds) > 0) - - ds = DataSet().read_pos("test/data_for_tests/people.txt") - self.assertTrue(isinstance(ds, DataSet)) - self.assertTrue(len(ds) > 0) - - def test_add_null(self): - # TODO test failed because 'fastNLP\core\fieldarray.py:143: RuntimeError' - ds = DataSet() - ds.add_field('test', []) - ds.set_target('test') + # def test_add_null(self): + # # TODO test failed because 'fastNLP\core\fieldarray.py:143: RuntimeError' + # ds = DataSet() + # ds.add_field('test', []) + # ds.set_target('test') class TestDataSetIter(unittest.TestCase): diff --git a/test/data_for_tests/sample_snli.jsonl b/test/data_for_tests/sample_snli.jsonl new file mode 100644 index 00000000..e62856ac --- /dev/null +++ b/test/data_for_tests/sample_snli.jsonl @@ -0,0 +1,3 @@ +{"annotator_labels": ["neutral"], "captionID": "3416050480.jpg#4", "gold_label": "neutral", "pairID": "3416050480.jpg#4r1n", "sentence1": "A person on a horse jumps over a broken down airplane.", "sentence1_binary_parse": "( ( ( A person ) ( on ( a horse ) ) ) ( ( jumps ( over ( a ( broken ( down airplane ) ) ) ) ) . ) )", "sentence1_parse": "(ROOT (S (NP (NP (DT A) (NN person)) (PP (IN on) (NP (DT a) (NN horse)))) (VP (VBZ jumps) (PP (IN over) (NP (DT a) (JJ broken) (JJ down) (NN airplane)))) (. .)))", "sentence2": "A person is training his horse for a competition.", "sentence2_binary_parse": "( ( A person ) ( ( is ( ( training ( his horse ) ) ( for ( a competition ) ) ) ) . ) )", "sentence2_parse": "(ROOT (S (NP (DT A) (NN person)) (VP (VBZ is) (VP (VBG training) (NP (PRP$ his) (NN horse)) (PP (IN for) (NP (DT a) (NN competition))))) (. .)))"} +{"annotator_labels": ["contradiction"], "captionID": "3416050480.jpg#4", "gold_label": "contradiction", "pairID": "3416050480.jpg#4r1c", "sentence1": "A person on a horse jumps over a broken down airplane.", "sentence1_binary_parse": "( ( ( A person ) ( on ( a horse ) ) ) ( ( jumps ( over ( a ( broken ( down airplane ) ) ) ) ) . ) )", "sentence1_parse": "(ROOT (S (NP (NP (DT A) (NN person)) (PP (IN on) (NP (DT a) (NN horse)))) (VP (VBZ jumps) (PP (IN over) (NP (DT a) (JJ broken) (JJ down) (NN airplane)))) (. .)))", "sentence2": "A person is at a diner, ordering an omelette.", "sentence2_binary_parse": "( ( A person ) ( ( ( ( is ( at ( a diner ) ) ) , ) ( ordering ( an omelette ) ) ) . ) )", "sentence2_parse": "(ROOT (S (NP (DT A) (NN person)) (VP (VBZ is) (PP (IN at) (NP (DT a) (NN diner))) (, ,) (S (VP (VBG ordering) (NP (DT an) (NN omelette))))) (. .)))"} +{"annotator_labels": ["entailment"], "captionID": "3416050480.jpg#4", "gold_label": "entailment", "pairID": "3416050480.jpg#4r1e", "sentence1": "A person on a horse jumps over a broken down airplane.", "sentence1_binary_parse": "( ( ( A person ) ( on ( a horse ) ) ) ( ( jumps ( over ( a ( broken ( down airplane ) ) ) ) ) . ) )", "sentence1_parse": "(ROOT (S (NP (NP (DT A) (NN person)) (PP (IN on) (NP (DT a) (NN horse)))) (VP (VBZ jumps) (PP (IN over) (NP (DT a) (JJ broken) (JJ down) (NN airplane)))) (. .)))", "sentence2": "A person is outdoors, on a horse.", "sentence2_binary_parse": "( ( A person ) ( ( ( ( is outdoors ) , ) ( on ( a horse ) ) ) . ) )", "sentence2_parse": "(ROOT (S (NP (DT A) (NN person)) (VP (VBZ is) (ADVP (RB outdoors)) (, ,) (PP (IN on) (NP (DT a) (NN horse)))) (. .)))"} \ No newline at end of file diff --git a/test/io/test_dataset_loader.py b/test/io/test_dataset_loader.py index 16e7d7ea..97379a7d 100644 --- a/test/io/test_dataset_loader.py +++ b/test/io/test_dataset_loader.py @@ -1,8 +1,7 @@ import unittest -from fastNLP.io.dataset_loader import Conll2003Loader, PeopleDailyCorpusLoader, ConllCWSReader, \ - ZhConllPOSReader, ConllxDataLoader - +from fastNLP.io.dataset_loader import Conll2003Loader, PeopleDailyCorpusLoader, \ + CSVLoader, SNLILoader class TestDatasetLoader(unittest.TestCase): @@ -17,11 +16,11 @@ class TestDatasetLoader(unittest.TestCase): def test_PeopleDailyCorpusLoader(self): data_set = PeopleDailyCorpusLoader().load("test/data_for_tests/people_daily_raw.txt") - def test_ConllCWSReader(self): - dataset = ConllCWSReader().load("test/data_for_tests/conll_example.txt") - - def test_ZhConllPOSReader(self): - dataset = ZhConllPOSReader().load("test/data_for_tests/zh_sample.conllx") + def test_CSVLoader(self): + ds = CSVLoader(sep='\t', headers=['words', 'label'])\ + .load('test/data_for_tests/tutorial_sample_dataset.csv') + assert len(ds) > 0 - def test_ConllxDataLoader(self): - dataset = ConllxDataLoader().load("test/data_for_tests/zh_sample.conllx") + def test_SNLILoader(self): + ds = SNLILoader().load('test/data_for_tests/sample_snli.jsonl') + assert len(ds) == 3 From abff8d9daadc160bcc5abe36d6f79b0cef0a7479 Mon Sep 17 00:00:00 2001 From: yunfan Date: Mon, 22 Apr 2019 13:57:28 +0800 Subject: [PATCH 7/9] - fix test_tutorial --- fastNLP/models/snli.py | 2 +- test/test_tutorials.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/fastNLP/models/snli.py b/fastNLP/models/snli.py index 5816d2af..901f2dd4 100644 --- a/fastNLP/models/snli.py +++ b/fastNLP/models/snli.py @@ -110,5 +110,5 @@ class ESIM(BaseModel): def predict(self, words1, words2, seq_len1, seq_len2): prediction = self.forward(words1, words2, seq_len1, seq_len2)['pred'] - return torch.argmax(prediction, dim=-1) + return {'pred': torch.argmax(prediction, dim=-1)} diff --git a/test/test_tutorials.py b/test/test_tutorials.py index bc0b5d2b..600699a3 100644 --- a/test/test_tutorials.py +++ b/test/test_tutorials.py @@ -379,6 +379,14 @@ class TestTutorial(unittest.TestCase): dev_data_2.apply(lambda x: [vocab_bert.to_index(word) for word in x['hypothesis']], new_field_name='hypothesis') train_data_2[-1], dev_data_2[-1] + for data in [train_data, dev_data, test_data]: + data.rename_field('premise', 'words1') + data.rename_field('hypothesis', 'words2') + data.rename_field('premise_len', 'seq_len1') + data.rename_field('hypothesis_len', 'seq_len2') + data.set_input('words1', 'words2', 'seq_len1', 'seq_len2') + + # step 1：加载模型参数（非必选） from fastNLP.io.config_io import ConfigSection, ConfigLoader args = ConfigSection() From 361a090c26509a21616c06ace42aee5aed423632 Mon Sep 17 00:00:00 2001 From: yh Date: Mon, 22 Apr 2019 14:07:21 +0800 Subject: [PATCH 8/9] =?UTF-8?q?=E6=B3=A8=E9=87=8A=E5=A2=9E=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/core/dataset.py | 46 +++++-- fastNLP/core/fieldarray.py | 241 +++++++++++++++++++++---------------- 2 files changed, 172 insertions(+), 115 deletions(-) diff --git a/fastNLP/core/dataset.py b/fastNLP/core/dataset.py index 013816a6..41858bdc 100644 --- a/fastNLP/core/dataset.py +++ b/fastNLP/core/dataset.py @@ -210,34 +210,62 @@ class DataSet(object): raise KeyError("DataSet has no field named {}.".format(old_name)) def set_target(self, *field_names, flag=True): - """Change the target flag of these fields. + """将field_names的target设置为flag状态 + Example:: - :param field_names: a sequence of str, indicating field names - :param bool flag: Set these fields as target if True. Unset them if False. + dataset.set_target('labels', 'seq_len') # 将labels和seq_len这两个field的target属性设置为True + dataset.set_target('labels', 'seq_lens', flag=False) # 将labels和seq_len的target属性设置为False + + :param field_names: str, field的名称 + :param flag: bool, 将field_name的target状态设置为flag """ + assert isinstance(flag, bool), "Only bool type supported." for name in field_names: if name in self.field_arrays: self.field_arrays[name].is_target = flag else: raise KeyError("{} is not a valid field name.".format(name)) - def set_input(self, *field_name, flag=True): - """Set the input flag of these fields. + def set_input(self, *field_names, flag=True): + """将field_name的input设置为flag状态 + Example:: + + dataset.set_input('words', 'seq_len') # 将words和seq_len这两个field的input属性设置为True + dataset.set_input('words', flag=False) # 将words这个field的input属性设置为False - :param field_name: a sequence of str, indicating field names. - :param bool flag: Set these fields as input if True. Unset them if False. + :param field_names: str, field的名称 + :param flag: bool, 将field_name的input状态设置为flag """ - for name in field_name: + for name in field_names: if name in self.field_arrays: self.field_arrays[name].is_input = flag else: raise KeyError("{} is not a valid field name.".format(name)) + def set_ignore_type(self, *field_names, flag=True): + """将field_names的ignore_type设置为flag状态 + + :param field_names: str, field的名称 + :param flag: bool, + :return: + """ + assert isinstance(flag, bool), "Only bool type supported." + for name in field_names: + if name in self.field_arrays: + self.field_arrays[name].ignore_type = flag + else: + raise KeyError("{} is not a valid field name.".format(name)) + def set_padder(self, field_name, padder): """为field_name设置padder + Example:: + + from fastNLP import EngChar2DPadder + padder = EngChar2DPadder() + dataset.set_padder('chars', padder) # 则chars这个field会使用EngChar2DPadder进行pad操作 :param field_name: str, 设置field的padding方式为padder - :param padder: PadderBase类型或None. 设置为None即删除padder。即对该field不进行padding操作. + :param padder: (None, PadderBase). 设置为None即删除padder, 即对该field不进行padding操作. :return: """ if field_name not in self.field_arrays: diff --git a/fastNLP/core/fieldarray.py b/fastNLP/core/fieldarray.py index c9caea56..3d0fb582 100644 --- a/fastNLP/core/fieldarray.py +++ b/fastNLP/core/fieldarray.py @@ -1,98 +1,6 @@ import numpy as np from copy import deepcopy -class PadderBase: - """ - 所有padder都需要继承这个类，并覆盖__call__()方法。 - 用于对batch进行padding操作。传入的element是inplace的，即直接修改element可能导致数据变化，建议inplace修改之前deepcopy一份。 - """ - def __init__(self, pad_val=0, **kwargs): - self.pad_val = pad_val - - def set_pad_val(self, pad_val): - self.pad_val = pad_val - - def __call__(self, contents, field_name, field_ele_dtype): - """ - 传入的是List内容。假设有以下的DataSet。 - from fastNLP import DataSet - from fastNLP import Instance - dataset = DataSet() - dataset.append(Instance(word='this is a demo', length=4, - chars=[['t', 'h', 'i', 's'], ['i', 's'], ['a'], ['d', 'e', 'm', 'o']])) - dataset.append(Instance(word='another one', length=2, - chars=[['a', 'n', 'o', 't', 'h', 'e', 'r'], ['o', 'n', 'e']])) - # 如果batch_size=2, 下面只是用str的方式看起来更直观一点，但实际上可能word和chars在pad时都已经为index了。 - word这个field的pad_func会接收到的内容会是 - [ - 'this is a demo', - 'another one' - ] - length这个field的pad_func会接收到的内容会是 - [4, 2] - chars这个field的pad_func会接收到的内容会是 - [ - [['t', 'h', 'i', 's'], ['i', 's'], ['a'], ['d', 'e', 'm', 'o']], - [['a', 'n', 'o', 't', 'h', 'e', 'r'], ['o', 'n', 'e']] - ] - 即把每个instance中某个field的内容合成一个List传入 - :param contents: List[element]。传入的element是inplace的，即直接修改element可能导致数据变化，建议inplace修改之前 - deepcopy一份。 - :param field_name: str, field的名称，帮助定位错误 - :param field_ele_dtype: np.int64, np.float64, np.str. 该field的内层list元素的类型。辅助判断是否pad，大多数情况用不上 - :return: List[padded_element]或np.array([padded_element]) - """ - raise NotImplementedError - - -class AutoPadder(PadderBase): - """ - 根据contents的数据自动判定是否需要做padding。 - - 1 如果元素类型(元素类型是指field中最里层List的元素的数据类型, 可以通过FieldArray.dtype查看，比如['This', 'is', ...]的元素类 - 型为np.str, [[1,2], ...]的元素类型为np.int64)的数据不为(np.int64, np.float64)则不会进行padding - - 2 如果元素类型为(np.int64, np.float64), - - 2.1 如果该field的内容只有一个，比如为sequence_length, 则不进行padding - - 2.2 如果该field的内容为List, 那么会将Batch中的List pad为一样长。若该List下还有里层的List需要padding，请使用其它padder。 - 如果某个instance中field为[1, 2, 3]，则可以pad； 若为[[1,2], [3,4, ...]]则不能进行pad - """ - def __init__(self, pad_val=0): - """ - :param pad_val: int, padding的位置使用该index - """ - super().__init__(pad_val=pad_val) - - def _is_two_dimension(self, contents): - """ - 判断contents是不是只有两个维度。[[1,2], [3]]是两个维度. [[[1,2], [3, 4, 5]], [[4,5]]]有三个维度 - :param contents: - :return: - """ - value = contents[0] - if isinstance(value , (np.ndarray, list)): - value = value[0] - if isinstance(value, (np.ndarray, list)): - return False - return True - return False - - def __call__(self, contents, field_name, field_ele_dtype): - if not is_iterable(contents[0]): - array = np.array([content for content in contents], dtype=field_ele_dtype) - elif field_ele_dtype in (np.int64, np.float64) and self._is_two_dimension(contents): - max_len = max([len(content) for content in contents]) - array = np.full((len(contents), max_len), self.pad_val, dtype=field_ele_dtype) - for i, content in enumerate(contents): - array[i][:len(content)] = content - elif field_ele_dtype is None: - array = np.array(contents) # 当ignore_type=True时，直接返回contents - else: # should only be str - array = np.array([content for content in contents]) - return array - class FieldArray(object): """``FieldArray`` is the collection of ``Instance``s of the same field. @@ -336,18 +244,18 @@ class FieldArray(object): self.content.append(val) def __getitem__(self, indices): - return self.get(indices) + return self.get(indices, pad=False) def __setitem__(self, idx, val): assert isinstance(idx, int) self.content[idx] = val def get(self, indices, pad=True): - """Fetch instances based on indices. + """根据给定的indices返回内容 - :param indices: an int, or a list of int. - :param pad: bool, 是否对返回的结果进行padding。 - :return: + :param indices: (int, List[int]), 获取indices对应的内容。 + :param pad: bool, 是否对返回的结果进行padding。仅对indices为List[int]时有效 + :return: (single, List) """ if isinstance(indices, int): return self.content[indices] @@ -362,14 +270,16 @@ class FieldArray(object): def set_padder(self, padder): """ - 设置padding方式 + 设置padder，在这个field进行pad的时候用这个padder进行pad，如果为None则不进行pad。 - :param padder: PadderBase类型或None. 设置为None即删除padder. + :param padder: (None, PadderBase). 设置为None即删除padder. :return: """ if padder is not None: assert isinstance(padder, PadderBase), "padder must be of type PadderBase." - self.padder = deepcopy(padder) + self.padder = deepcopy(padder) + else: + self.padder = None def set_pad_val(self, pad_val): """修改padder的pad_val. @@ -391,7 +301,7 @@ class FieldArray(object): def to(self, other): """ - 将other的属性复制给本FieldArray(other必须为FieldArray类型). 包含 is_input, is_target, padder, ignore_type + 将other的属性复制给本FieldArray(other必须为FieldArray类型).属性包括 is_input, is_target, padder, ignore_type :param other: FieldArray :return: @@ -413,17 +323,136 @@ def is_iterable(content): return True +class PadderBase: + """ + 所有padder都需要继承这个类，并覆盖__call__()方法。 + 用于对batch进行padding操作。传入的element是inplace的，即直接修改element可能导致数据变化，建议inplace修改之前deepcopy一份。 + """ + + def __init__(self, pad_val=0, **kwargs): + self.pad_val = pad_val + + def set_pad_val(self, pad_val): + self.pad_val = pad_val + + def __call__(self, contents, field_name, field_ele_dtype): + """ + 传入的是List内容。假设有以下的DataSet。 + + :param contents: List[element]。传入的element是inplace的，即直接修改element可能导致数据变化，建议inplace修改之前 + deepcopy一份。 + :param field_name: str, field的名称。 + :param field_ele_dtype: (np.int64, np.float64, np.str, None), 该field的内层元素的类型。如果该field的ignore_type + 为True，该这个值为None。 + :return: np.array([padded_element]) + + Example:: + + from fastNLP import DataSet + from fastNLP import Instance + dataset = DataSet() + dataset.append(Instance(sent='this is a demo', length=4, + chars=[['t', 'h', 'i', 's'], ['i', 's'], ['a'], ['d', 'e', 'm', 'o']])) + dataset.append(Instance(sent='another one', length=2, + chars=[['a', 'n', 'o', 't', 'h', 'e', 'r'], ['o', 'n', 'e']])) + 如果调用 + batch = dataset.get([0,1], pad=True) + sent这个field的padder的__call__会接收到的内容会是 + [ + 'this is a demo', + 'another one' + ] + + length这个field的padder的__call__会接收到的内容会是 + [4, 2] + + chars这个field的padder的__call__会接收到的内容会是 + [ + [['t', 'h', 'i', 's'], ['i', 's'], ['a'], ['d', 'e', 'm', 'o']], + [['a', 'n', 'o', 't', 'h', 'e', 'r'], ['o', 'n', 'e']] + ] + + 即把每个instance中某个field的内容合成一个List传入 + + """ + raise NotImplementedError + + +class AutoPadder(PadderBase): + """ + 根据contents的数据自动判定是否需要做padding。 + + 1 如果元素类型(元素类型是指field中最里层元素的数据类型, 可以通过FieldArray.dtype查看，比如['This', 'is', ...]的元素类 + 型为np.str, [[1,2], ...]的元素类型为np.int64)的数据不为(np.int64, np.float64)则不会进行pad + + 2 如果元素类型为(np.int64, np.float64), + + 2.1 如果该field的内容为(np.int64, np.float64)，比如为seq_len, 则不进行padding + + 2.2 如果该field的内容为List, 那么会将Batch中的List pad为一样长。若该List下还有里层的List需要padding，请使用其它padder。 + 如果某个instance中field为[1, 2, 3]，则可以pad；若为[[1,2], [3,4, ...]]则不能进行pad + """ + + def __init__(self, pad_val=0): + """ + :param pad_val: int, padding的位置使用该index + """ + super().__init__(pad_val=pad_val) + + def _is_two_dimension(self, contents): + """ + 判断contents是不是只有两个维度。[[1,2], [3]]是两个维度. [[[1,2], [3, 4, 5]], [[4,5]]]有三个维度 + :param contents: + :return: + """ + value = contents[0] + if isinstance(value, (np.ndarray, list)): + value = value[0] + if isinstance(value, (np.ndarray, list)): + return False + return True + return False + + def __call__(self, contents, field_name, field_ele_dtype): + if not is_iterable(contents[0]): + array = np.array([content for content in contents], dtype=field_ele_dtype) + elif field_ele_dtype in (np.int64, np.float64) and self._is_two_dimension(contents): + max_len = max([len(content) for content in contents]) + array = np.full((len(contents), max_len), self.pad_val, dtype=field_ele_dtype) + for i, content in enumerate(contents): + array[i][:len(content)] = content + elif field_ele_dtype is None: + array = np.array(contents) # 当ignore_type=True时，直接返回contents + else: # should only be str + array = np.array([content for content in contents]) + return array + + class EngChar2DPadder(PadderBase): """ - 用于为英语执行character级别的2D padding操作。对应的field内容应该为[['T', 'h', 'i', 's'], ['a'], ['d', 'e', 'm', 'o']](这里为 - 了更直观，把它们写为str，但实际使用时它们应该是character的index)。 - padded过后的batch内容，形状为(batch_size, max_sentence_length, max_word_length). max_sentence_length最大句子长度。 - max_word_length最长的word的长度 + 用于为英语执行character级别的2D padding操作。对应的field内容应该类似[['T', 'h', 'i', 's'], ['a'], ['d', 'e', 'm', 'o']]， + 但这个Padder只能处理index为int的情况。 + padded过后的batch内容，形状为(batch_size, max_sentence_length, max_word_length). max_sentence_length为这个batch中最大句 + 子长度；max_word_length为这个batch中最长的word的长度 + + Example:: + + from fastNLP import DataSet + from fastNLP import EnChar2DPadder + from fastNLP import Vocabulary + dataset = DataSet({'sent': ['This is the first demo', 'This is the second demo']}) + dataset.apply(lambda ins:[list(word) for word in ins['sent'].split()], new_field_name='chars') + vocab = Vocabulary() + vocab.from_dataset(dataset, field_name='chars') + vocab.index_dataset(dataset, field_name='chars') + dataset.set_input('chars') + padder = EnChar2DPadder() + dataset.set_padder('chars', padder) # chars这个field的设置为了EnChar2DPadder """ def __init__(self, pad_val=0, pad_length=0): """ - :param pad_val: int, padding的位置使用该index + :param pad_val: int, pad的位置使用该index :param pad_length: int, 如果为0则取一个batch中最大的单词长度作为padding长度。如果为大于0的数，则将所有单词的长度都pad或截 取到该长度. """ From 2c202bb1516d7d49c24cacca98c38c6a35c46583 Mon Sep 17 00:00:00 2001 From: yh Date: Mon, 22 Apr 2019 17:18:28 +0800 Subject: [PATCH 9/9] =?UTF-8?q?=E6=B5=8B=E8=AF=95=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/fastNLP.io.rst | 8 ++ docs/source/fastNLP.modules.decoder.rst | 8 ++ fastNLP/core/dataset.py | 168 ++++++++++++++---------- fastNLP/core/fieldarray.py | 20 +-- test/core/test_dataset.py | 5 + 5 files changed, 133 insertions(+), 76 deletions(-) diff --git a/docs/source/fastNLP.io.rst b/docs/source/fastNLP.io.rst index bb30c5e7..e73f27d3 100644 --- a/docs/source/fastNLP.io.rst +++ b/docs/source/fastNLP.io.rst @@ -36,6 +36,14 @@ fastNLP.io.embed\_loader module :undoc-members: :show-inheritance: +fastNLP.io.file\_reader module +------------------------------ + +.. automodule:: fastNLP.io.file_reader + :members: + :undoc-members: + :show-inheritance: + fastNLP.io.model\_io module --------------------------- diff --git a/docs/source/fastNLP.modules.decoder.rst b/docs/source/fastNLP.modules.decoder.rst index 25602b2c..60706b06 100644 --- a/docs/source/fastNLP.modules.decoder.rst +++ b/docs/source/fastNLP.modules.decoder.rst @@ -20,6 +20,14 @@ fastNLP.modules.decoder.MLP module :undoc-members: :show-inheritance: +fastNLP.modules.decoder.utils module +------------------------------------ + +.. automodule:: fastNLP.modules.decoder.utils + :members: + :undoc-members: + :show-inheritance: + Module contents --------------- diff --git a/fastNLP/core/dataset.py b/fastNLP/core/dataset.py index 5bf32d02..3a4dfa55 100644 --- a/fastNLP/core/dataset.py +++ b/fastNLP/core/dataset.py @@ -1,3 +1,18 @@ +""" +fastNLP.core.DataSet的介绍文档 + +DataSet是fastNLP中用于承载数据的容器。可以将DataSet看做是一个表格，每一行是一个instance(或sample)，每一列是一个feature。 + +csv-table:: +:header: "Field1", "Field2", "Field3" +:widths:20, 10, 10 + +"This is the first instance", ['This', 'is', 'the', 'first', 'instance'], 5 +"Second instance", ['Second', 'instance'], 2 + +""" + + import _pickle as pickle import numpy as np @@ -31,7 +46,7 @@ class DataSet(object): length_set.add(len(value)) assert len(length_set) == 1, "Arrays must all be same length." for key, value in data.items(): - self.add_field(name=key, fields=value) + self.add_field(field_name=key, fields=value) elif isinstance(data, list): for ins in data: assert isinstance(ins, Instance), "Must be Instance type, not {}.".format(type(ins)) @@ -88,7 +103,7 @@ class DataSet(object): raise RuntimeError(f"Start index {idx.start} out of range 0-{len(self)-1}") data_set = DataSet() for field in self.field_arrays.values(): - data_set.add_field(name=field.name, fields=field.content[idx], padder=field.padder, + data_set.add_field(field_name=field.name, fields=field.content[idx], padder=field.padder, is_input=field.is_input, is_target=field.is_target, ignore_type=field.ignore_type) return data_set elif isinstance(idx, str): @@ -131,7 +146,7 @@ class DataSet(object): return "DataSet(" + self.__inner_repr__() + ")" def append(self, ins): - """Add an instance to the DataSet. + """将一个instance对象append到DataSet后面。 If the DataSet is not empty, the instance must have the same field names as the rest instances in the DataSet. :param ins: an Instance object @@ -151,57 +166,60 @@ class DataSet(object): assert name in self.field_arrays self.field_arrays[name].append(field) - def add_field(self, name, fields, padder=None, is_input=False, is_target=False, ignore_type=False): - """Add a new field to the DataSet. + def add_field(self, field_name, fields, padder=AutoPadder(), is_input=False, is_target=False, ignore_type=False): + """新增一个field - :param str name: the name of the field. - :param fields: a list of int, float, or other objects. - :param padder: PadBase对象，如何对该Field进行padding。如果为None则使用 - :param bool is_input: whether this field is model input. - :param bool is_target: whether this field is label or target. - :param bool ignore_type: If True, do not perform type check. (Default: False) + :param str field_name: 新增的field的名称 + :param list fields: 需要新增的field的内容 + :param None, Padder padder: 如果为None,则不进行pad。 + :param bool is_input: 新加入的field是否是input + :param bool is_target: 新加入的field是否是target + :param bool ignore_type: 是否忽略对新加入的field的类型检查 """ - if padder is None: - padder = AutoPadder(pad_val=0) if len(self.field_arrays) != 0: if len(self) != len(fields): raise RuntimeError(f"The field to append must have the same size as dataset. " f"Dataset size {len(self)} != field size {len(fields)}") - self.field_arrays[name] = FieldArray(name, fields, is_target=is_target, is_input=is_input, - padder=padder, ignore_type=ignore_type) + self.field_arrays[field_name] = FieldArray(field_name, fields, is_target=is_target, is_input=is_input, + padder=padder, ignore_type=ignore_type) - def delete_field(self, name): - """Delete a field based on the field name. + def delete_field(self, field_name): + """删除field - :param name: the name of the field to be deleted. + :param str field_name: 需要删除的field的名称. """ - self.field_arrays.pop(name) + self.field_arrays.pop(field_name) def get_field(self, field_name): + """获取field_name这个field + + :param str field_name: field的名称 + :return: FieldArray + """ if field_name not in self.field_arrays: raise KeyError("Field name {} not found in DataSet".format(field_name)) return self.field_arrays[field_name] def get_all_fields(self): - """Return all the fields with their names. + """返回一个dict，key为field_name, value为对应的FieldArray - :return field_arrays: the internal data structure of DataSet. + :return: dict: """ return self.field_arrays def get_length(self): - """Fetch the length of the dataset. + """获取DataSet的元素数量 - :return length: + :return: int length: """ return len(self) def rename_field(self, old_name, new_name): - """Rename a field. + """将某个field重新命名. - :param str old_name: - :param str new_name: + :param str old_name: 原来的field名称 + :param str new_name: 修改为new_name """ if old_name in self.field_arrays: self.field_arrays[new_name] = self.field_arrays.pop(old_name) @@ -216,8 +234,8 @@ class DataSet(object): dataset.set_target('labels', 'seq_len') # 将labels和seq_len这两个field的target属性设置为True dataset.set_target('labels', 'seq_lens', flag=False) # 将labels和seq_len的target属性设置为False - :param field_names: str, field的名称 - :param flag: bool, 将field_name的target状态设置为flag + :param str field_names: field的名称 + :param bool flag: 将field_name的target状态设置为flag """ assert isinstance(flag, bool), "Only bool type supported." for name in field_names: @@ -233,8 +251,8 @@ class DataSet(object): dataset.set_input('words', 'seq_len') # 将words和seq_len这两个field的input属性设置为True dataset.set_input('words', flag=False) # 将words这个field的input属性设置为False - :param field_names: str, field的名称 - :param flag: bool, 将field_name的input状态设置为flag + :param str field_names: field的名称 + :param bool flag: 将field_name的input状态设置为flag """ for name in field_names: if name in self.field_arrays: @@ -245,8 +263,8 @@ class DataSet(object): def set_ignore_type(self, *field_names, flag=True): """将field_names的ignore_type设置为flag状态 - :param field_names: str, field的名称 - :param flag: bool, + :param str field_names: field的名称 + :param bool flag: 将field_name的ignore_type状态设置为flag :return: """ assert isinstance(flag, bool), "Only bool type supported." @@ -264,8 +282,8 @@ class DataSet(object): padder = EngChar2DPadder() dataset.set_padder('chars', padder) # 则chars这个field会使用EngChar2DPadder进行pad操作 - :param field_name: str, 设置field的padding方式为padder - :param padder: (None, PadderBase). 设置为None即删除padder, 即对该field不进行padding操作. + :param str field_name: 设置field的padding方式为padder + :param None, Padder padder: 设置为None即删除padder, 即对该field不进行pad操作. :return: """ if field_name not in self.field_arrays: @@ -275,8 +293,8 @@ class DataSet(object): def set_pad_val(self, field_name, pad_val): """为某个field设置对应的pad_val. - :param field_name: str，修改该field的pad_val - :param pad_val: int，该field的padder会以pad_val作为padding index + :param str field_name: 修改该field的pad_val + :param int pad_val: 该field的padder会以pad_val作为padding index :return: """ if field_name not in self.field_arrays: @@ -286,7 +304,7 @@ class DataSet(object): def get_input_name(self): """返回所有is_input被设置为True的field名称 - :return list, 里面的元素为被设置为input的field名称 + :return: list, 里面的元素为被设置为input的field名称 """ return [name for name, field in self.field_arrays.items() if field.is_input] @@ -300,15 +318,22 @@ class DataSet(object): def apply_field(self, func, field_name, new_field_name=None, **kwargs): """将DataSet中的每个instance中的`field_name`这个field传给func，并获取它的返回值. - :param func: Callable, input是instance的`field_name`这个field. - :param field_name: str, 传入func的是哪个field. - :param new_field_name: (str, None). 如果不是None，将func的返回值放入这个名为`new_field_name`的新field中，如果名称与已有 - 的field相同，则覆盖之前的field. - :param **kwargs: 合法的参数有以下三个 - (1) is_input: bool, 如果为True则将`new_field_name`这个field设置为input - (2) is_target: bool, 如果为True则将`new_field_name`这个field设置为target - (3) ignore_type: bool, 如果为True则将`new_field_name`这个field的ignore_type设置为true, 忽略其类型 - :return: List[], 里面的元素为func的返回值，所以list长度为DataSet的长度 + :param callable func: input是instance的`field_name`这个field. + :param str field_name: 传入func的是哪个field. + :param str, None new_field_name: 将func返回的内容放入到什么field中 + + 1. str, 将func的返回值放入这个名为`new_field_name`的新field中，如果名称与已有的field相 + 同，则覆盖之前的field + + 2. None, 不创建新的field + :param kwargs: 合法的参数有以下三个 + + 1. is_input: bool, 如果为True则将`new_field_name`的field设置为input + + 2. is_target: bool, 如果为True则将`new_field_name`的field设置为target + + 3. ignore_type: bool, 如果为True则将`new_field_name`的field的ignore_type设置为true, 忽略其类型 + :return: list(Any), 里面的元素为func的返回值，所以list长度为DataSet的长度 """ assert len(self)!=0, "Null DataSet cannot use apply()." @@ -334,9 +359,9 @@ class DataSet(object): def _add_apply_field(self, results, new_field_name, kwargs): """将results作为加入到新的field中，field名称为new_field_name - :param results: List[], 一般是apply*()之后的结果 - :param new_field_name: str, 新加入的field的名称 - :param kwargs: dict, 用户apply*()时传入的自定义参数 + :param list(str) results: 一般是apply*()之后的结果 + :param str new_field_name: 新加入的field的名称 + :param dict kwargs: 用户apply*()时传入的自定义参数 :return: """ extra_param = {} @@ -355,23 +380,30 @@ class DataSet(object): extra_param['is_target'] = old_field.is_target if 'ignore_type' not in extra_param: extra_param['ignore_type'] = old_field.ignore_type - self.add_field(name=new_field_name, fields=results, is_input=extra_param["is_input"], + self.add_field(field_name=new_field_name, fields=results, is_input=extra_param["is_input"], is_target=extra_param["is_target"], ignore_type=extra_param['ignore_type']) else: - self.add_field(name=new_field_name, fields=results, is_input=extra_param.get("is_input", None), + self.add_field(field_name=new_field_name, fields=results, is_input=extra_param.get("is_input", None), is_target=extra_param.get("is_target", None), ignore_type=extra_param.get("ignore_type", False)) def apply(self, func, new_field_name=None, **kwargs): """将DataSet中每个instance传入到func中，并获取它的返回值. - :param func: Callable, 参数是DataSet中的instance - :param new_field_name: (None, str). (1) None, 不创建新的field; (2) str，将func的返回值放入这个名为 - `new_field_name`的新field中，如果名称与已有的field相同，则覆盖之前的field; + :param callable func: 参数是DataSet中的instance + :param str, None new_field_name: 将func返回的内容放入到什么field中 + + 1. str, 将func的返回值放入这个名为`new_field_name`的新field中，如果名称与已有的field相 + 同，则覆盖之前的field + + 2. None, 不创建新的field :param kwargs: 合法的参数有以下三个 - (1) is_input: bool, 如果为True则将`new_field_name`的field设置为input - (2) is_target: bool, 如果为True则将`new_field_name`的field设置为target - (3) ignore_type: bool, 如果为True则将`new_field_name`的field的ignore_type设置为true, 忽略其类型 + + 1. is_input: bool, 如果为True则将`new_field_name`的field设置为input + + 2. is_target: bool, 如果为True则将`new_field_name`的field设置为target + + 3. ignore_type: bool, 如果为True则将`new_field_name`的field的ignore_type设置为true, 忽略其类型 :return: List[], 里面的元素为func的返回值，所以list长度为DataSet的长度 """ assert len(self)!=0, "Null DataSet cannot use apply()." @@ -396,10 +428,10 @@ class DataSet(object): def drop(self, func, inplace=True): """func接受一个instance，返回bool值，返回值为True时，该instance会被删除。 - :param func: Callable, 接受一个instance作为参数，返回bool值。为True时删除该instance - :param inplace: bool, 是否在当前DataSet中直接删除instance。如果为False，返回值为一个删除了相应instance的新的DataSet + :param callable func: 接受一个instance作为参数，返回bool值。为True时删除该instance + :param bool inplace: 是否在当前DataSet中直接删除instance。如果为False，返回值为一个删除了相应instance的新的DataSet - :return: DataSet. + :return: DataSet """ if inplace: results = [ins for ins in self._inner_iter() if not func(ins)] @@ -408,16 +440,16 @@ class DataSet(object): return self else: results = [ins for ins in self if not func(ins)] - data = DataSet(results) + dataset = DataSet(results) for field_name, field in self.field_arrays.items(): - data.field_arrays[field_name].to(field) - return data + dataset.field_arrays[field_name].to(field) + return dataset def split(self, ratio): """将DataSet按照ratio的比例拆分，返回两个DataSet - :param ratio: float, 0