From 999a14381747068e9e6a7cc370037b320197db00 Mon Sep 17 00:00:00 2001 From: ChenXin Date: Wed, 5 Jun 2019 20:13:23 +0800 Subject: [PATCH 001/141] fix a bug in metrics.py --- fastNLP/core/metrics.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/fastNLP/core/metrics.py b/fastNLP/core/metrics.py index f994bd31..868d67b1 100644 --- a/fastNLP/core/metrics.py +++ b/fastNLP/core/metrics.py @@ -438,7 +438,7 @@ def _bio_tag_to_spans(tags, ignore_labels=None): class SpanFPreRecMetric(MetricBase): - """ + r""" 别名：:class:`fastNLP.SpanFPreRecMetric` :class:`fastNLP.core.metrics.SpanFPreRecMetric` 在序列标注问题中，以span的方式计算F, pre, rec. @@ -476,7 +476,7 @@ class SpanFPreRecMetric(MetricBase): label的f1, pre, rec :param str f_type: 'micro'或'macro'. 'micro':通过先计算总体的TP，FN和FP的数量，再计算f, precision, recall; 'macro': 分布计算每个类别的f, precision, recall，然后做平均（各类别f的权重相同） - :param float beta: f_beta分数，:math:`f_beta = \frac{(1 + {beta}^{2})*(pre*rec)}{({beta}^{2}*pre + rec)}`. + :param float beta: f_beta分数， :math:`f_{beta} = \frac{(1 + {beta}^{2})*(pre*rec)}{({beta}^{2}*pre + rec)}` . 常用为beta=0.5, 1, 2. 若为0.5则精确率的权重高于召回率；若为1，则两者平等；若为2，则召回率权重高于精确率。 """ @@ -699,16 +699,16 @@ def _pred_topk(y_prob, k=1): class SQuADMetric(MetricBase): - """ + r""" 别名：:class:`fastNLP.SQuADMetric` :class:`fastNLP.core.metrics.SQuADMetric` SQuAD数据集metric - :param pred1: 参数映射表中`pred1`的映射关系，None表示映射关系为`pred1`->`pred1` - :param pred2: 参数映射表中`pred2`的映射关系，None表示映射关系为`pred2`->`pred2` - :param target1: 参数映射表中`target1`的映射关系，None表示映射关系为`target1`->`target1` - :param target2: 参数映射表中`target2`的映射关系，None表示映射关系为`target2`->`target2` - :param float beta: f_beta分数，:math:`f_beta = \frac{(1 + {beta}^{2})*(pre*rec)}{({beta}^{2}*pre + rec)}`. + :param pred1: 参数映射表中 `pred1` 的映射关系，None表示映射关系为 `pred1` -> `pred1` + :param pred2: 参数映射表中 `pred2` 的映射关系，None表示映射关系为 `pred2` -> `pred2` + :param target1: 参数映射表中 `target1` 的映射关系，None表示映射关系为 `target1` -> `target1` + :param target2: 参数映射表中 `target2` 的映射关系，None表示映射关系为 `target2` -> `target2` + :param float beta: f_beta分数， :math:`f_{beta} = \frac{(1 + {beta}^{2})*(pre*rec)}{({beta}^{2}*pre + rec)}` . 常用为beta=0.5, 1, 2. 若为0.5则精确率的权重高于召回率；若为1，则两者平等；若为2，则召回率权重高于精确率。 :param bool right_open: right_open为true表示start跟end指针指向一个左闭右开区间，为false表示指向一个左闭右闭区间。 :param bool print_predict_stat: True则输出预测答案是否为空与正确答案是否为空的统计信息, False则不输出 From 807a8f1fb34dffc91309840d3155837a8dc2aa10 Mon Sep 17 00:00:00 2001 From: yh Date: Fri, 12 Jul 2019 09:27:11 +0800 Subject: [PATCH 002/141] =?UTF-8?q?1.=20BucketSampler=E4=B8=8D=E9=9C=80?= =?UTF-8?q?=E8=A6=81=E8=87=AA=E5=B7=B1=E4=BC=A0=E5=85=A5batch=5Fsize?= =?UTF-8?q?=E4=BA=86=EF=BC=8C=E7=94=B1Trainer=E8=87=AA=E5=8A=A8=E8=AE=BE?= =?UTF-8?q?=E7=BD=AE=202.=20sequence=20labeling=E4=B8=AD=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?BiLSTMCRF=203.=20embedding=E4=B8=AD=E4=BF=AE=E5=A4=8Dbug=204.?= =?UTF-8?q?=20=E5=B0=86=E4=B9=8B=E5=89=8D=E7=89=88=E6=9C=AC=E7=9A=84reprod?= =?UTF-8?q?uction=E7=A7=BB=E5=8A=A8=E5=88=B0legacy=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E5=A4=B9=205.=20sequence=20labeling=E4=B8=AD=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E4=B8=AD=E6=96=87NER=206.=20text=20classification=E4=B8=ADload?= =?UTF-8?q?er=E5=A2=9E=E5=BC=BA=EF=BC=8C=E4=BF=AE=E6=94=B9lstm=E6=A8=A1?= =?UTF-8?q?=E5=9E=8B=EF=BC=8C=E5=A2=9E=E5=8A=A0train=5Fbert.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/core/sampler.py | 17 ++- fastNLP/core/trainer.py | 2 + fastNLP/io/base_loader.py | 2 +- fastNLP/models/sequence_labeling.py | 61 +++++++++- fastNLP/modules/encoder/embedding.py | 2 +- fastNLP/modules/encoder/lstm.py | 4 - .../{ => legacy}/Biaffine_parser/cfg.cfg | 0 .../{ => legacy}/Biaffine_parser/infer.py | 0 .../{ => legacy}/Biaffine_parser/main.py | 0 .../{ => legacy}/Biaffine_parser/run.py | 0 .../{ => legacy}/Biaffine_parser/util.py | 0 .../Chinese_word_segmentation/__init__.py | 0 .../Chinese_word_segmentation/cws.cfg | 0 .../cws_io/__init__.py | 0 .../cws_io/cws_reader.py | 0 .../models/__init__.py | 0 .../models/cws_model.py | 2 +- .../models/cws_transformer.py | 4 +- .../process/__init__.py | 0 .../process/cws_processor.py | 2 +- .../process/span_converter.py | 0 .../Chinese_word_segmentation/utils.py | 0 .../README.md | 0 .../Word2Idx.py | 0 .../config.cfg | 0 .../dataloader.py | 0 .../example.py | 0 .../main.py | 0 .../predict.py | 0 .../prepare.py | 0 .../{ => legacy}/POS_tagging/pos_processor.py | 0 .../{ => legacy}/POS_tagging/pos_reader.py | 0 .../{ => legacy}/POS_tagging/pos_tag.cfg | 0 .../{ => legacy}/POS_tagging/train_pos_tag.py | 0 .../{ => legacy}/POS_tagging/utils.py | 0 .../chinese_ner/data/ChineseNER.py | 115 ++++++++++++++++++ .../chinese_ner/data/__init__.py | 0 .../chinese_ner/train_cn_ner.py | 94 ++++++++++++++ .../seqence_labelling/ner/train_ontonote.py | 76 +++++++----- .../text_classification/data/IMDBLoader.py | 2 +- .../text_classification/model/lstm.py | 8 +- .../text_classification/train_bert.py | 33 +++++ 42 files changed, 376 insertions(+), 48 deletions(-) rename reproduction/{ => legacy}/Biaffine_parser/cfg.cfg (100%) rename reproduction/{ => legacy}/Biaffine_parser/infer.py (100%) rename reproduction/{ => legacy}/Biaffine_parser/main.py (100%) rename reproduction/{ => legacy}/Biaffine_parser/run.py (100%) rename reproduction/{ => legacy}/Biaffine_parser/util.py (100%) rename reproduction/{ => legacy}/Chinese_word_segmentation/__init__.py (100%) rename reproduction/{ => legacy}/Chinese_word_segmentation/cws.cfg (100%) rename reproduction/{ => legacy}/Chinese_word_segmentation/cws_io/__init__.py (100%) rename reproduction/{ => legacy}/Chinese_word_segmentation/cws_io/cws_reader.py (100%) rename reproduction/{ => legacy}/Chinese_word_segmentation/models/__init__.py (100%) rename reproduction/{ => legacy}/Chinese_word_segmentation/models/cws_model.py (98%) rename reproduction/{ => legacy}/Chinese_word_segmentation/models/cws_transformer.py (97%) rename reproduction/{ => legacy}/Chinese_word_segmentation/process/__init__.py (100%) rename reproduction/{ => legacy}/Chinese_word_segmentation/process/cws_processor.py (99%) rename reproduction/{ => legacy}/Chinese_word_segmentation/process/span_converter.py (100%) rename reproduction/{ => legacy}/Chinese_word_segmentation/utils.py (100%) rename reproduction/{ => legacy}/LSTM+self_attention_sentiment_analysis/README.md (100%) rename reproduction/{ => legacy}/LSTM+self_attention_sentiment_analysis/Word2Idx.py (100%) rename reproduction/{ => legacy}/LSTM+self_attention_sentiment_analysis/config.cfg (100%) rename reproduction/{ => legacy}/LSTM+self_attention_sentiment_analysis/dataloader.py (100%) rename reproduction/{ => legacy}/LSTM+self_attention_sentiment_analysis/example.py (100%) rename reproduction/{ => legacy}/LSTM+self_attention_sentiment_analysis/main.py (100%) rename reproduction/{ => legacy}/LSTM+self_attention_sentiment_analysis/predict.py (100%) rename reproduction/{ => legacy}/LSTM+self_attention_sentiment_analysis/prepare.py (100%) rename reproduction/{ => legacy}/POS_tagging/pos_processor.py (100%) rename reproduction/{ => legacy}/POS_tagging/pos_reader.py (100%) rename reproduction/{ => legacy}/POS_tagging/pos_tag.cfg (100%) rename reproduction/{ => legacy}/POS_tagging/train_pos_tag.py (100%) rename reproduction/{ => legacy}/POS_tagging/utils.py (100%) create mode 100644 reproduction/seqence_labelling/chinese_ner/data/ChineseNER.py create mode 100644 reproduction/seqence_labelling/chinese_ner/data/__init__.py create mode 100644 reproduction/seqence_labelling/chinese_ner/train_cn_ner.py diff --git a/fastNLP/core/sampler.py b/fastNLP/core/sampler.py index c5784f59..d8ba1ad1 100644 --- a/fastNLP/core/sampler.py +++ b/fastNLP/core/sampler.py @@ -62,16 +62,27 @@ class BucketSampler(Sampler): 带Bucket的 `Random Sampler`. 可以随机地取出长度相似的元素 :param int num_buckets: bucket的数量 - :param int batch_size: batch的大小 + :param int batch_size: batch的大小. 默认为None，Trainer在调用BucketSampler时，会将该值正确设置，如果是非Trainer场景使用，需 + 要显示传递该值 :param str seq_len_field_name: 对应序列长度的 `field` 的名字 """ - def __init__(self, num_buckets=10, batch_size=32, seq_len_field_name='seq_len'): + def __init__(self, num_buckets=10, batch_size=None, seq_len_field_name='seq_len'): self.num_buckets = num_buckets self.batch_size = batch_size self.seq_len_field_name = seq_len_field_name - + + def set_batch_size(self, batch_size): + """ + + :param int batch_size: 每个batch的大小 + :return: + """ + self.batch_size = batch_size + def __call__(self, data_set): + if self.batch_size is None: + raise RuntimeError("batch_size is None.") seq_lens = data_set.get_all_fields()[self.seq_len_field_name].content total_sample_num = len(seq_lens) diff --git a/fastNLP/core/trainer.py b/fastNLP/core/trainer.py index eabda99c..8fa44438 100644 --- a/fastNLP/core/trainer.py +++ b/fastNLP/core/trainer.py @@ -443,6 +443,8 @@ class Trainer(object): if sampler is None: sampler = RandomSampler() + elif hasattr(sampler, 'set_batch_size'): + sampler.set_batch_size(batch_size) if isinstance(train_data, DataSet): self.data_iterator = DataSetIter( diff --git a/fastNLP/io/base_loader.py b/fastNLP/io/base_loader.py index 62793836..5d61c16a 100644 --- a/fastNLP/io/base_loader.py +++ b/fastNLP/io/base_loader.py @@ -111,7 +111,7 @@ def _uncompress(src, dst): class DataBundle: """ - 经过处理的数据信息，包括一系列数据集（比如：分开的训练集、验证集和测试集）及它们所用的词表和词嵌入。 + 经过处理的数据信息，包括一系列数据集（比如：分开的训练集、验证集和测试集）以及各个field对应的vocabulary。 :param vocabs: 从名称(字符串)到 :class:`~fastNLP.Vocabulary` 类型的dict :param datasets: 从名称(字符串)到 :class:`~fastNLP.DataSet` 类型的dict diff --git a/fastNLP/models/sequence_labeling.py b/fastNLP/models/sequence_labeling.py index 8e6a5db1..506ebdc6 100644 --- a/fastNLP/models/sequence_labeling.py +++ b/fastNLP/models/sequence_labeling.py @@ -3,17 +3,76 @@ """ __all__ = [ "SeqLabeling", - "AdvSeqLabel" + "AdvSeqLabel", + "BiLSTMCRF" ] import torch import torch.nn as nn +import torch.nn.functional as F from .base_model import BaseModel from ..modules import decoder, encoder from ..modules.decoder.crf import allowed_transitions from ..core.utils import seq_len_to_mask from ..core.const import Const as C +from ..modules import LSTM +from ..modules import get_embeddings +from ..modules import ConditionalRandomField + + +class BiLSTMCRF(BaseModel): + """ + 结构为BiLSTM + FC + Dropout + CRF. + TODO 补充文档 + :param embed: tuple: + :param num_classes: + :param num_layers: + :param hidden_size: + :param dropout: + :param target_vocab: + :param encoding_type: + """ + def __init__(self, embed, num_classes, num_layers=1, hidden_size=100, dropout=0.5, + target_vocab=None, encoding_type=None): + super().__init__() + self.embed = get_embeddings(embed) + + if num_layers>1: + self.lstm = LSTM(embed.embedding_dim, num_layers=num_layers, hidden_size=hidden_size, bidirectional=True, + batch_first=True, dropout=dropout) + else: + self.lstm = LSTM(embed.embedding_dim, num_layers=num_layers, hidden_size=hidden_size, bidirectional=True, + batch_first=True) + + self.dropout = nn.Dropout(dropout) + self.fc = nn.Linear(hidden_size, num_classes) + + trans = None + if target_vocab is not None and encoding_type is not None: + trans = allowed_transitions(target_vocab.idx2word, encoding_type=encoding_type, include_start_end=True) + + self.crf = ConditionalRandomField(num_classes, include_start_end_trans=True, allowed_transitions=trans) + + def _forward(self, words, seq_len=None, target=None): + words = self.embed(words) + feats = self.lstm(words, seq_len=seq_len) + feats = self.fc(feats) + feats = self.dropout(feats) + logits = F.log_softmax(feats, dim=-1) + mask = seq_len_to_mask(seq_len) + if target is None: + pred, _ = self.crf.viterbi_decode(logits, mask) + return {C.OUTPUT:pred} + else: + loss = self.crf(logits, target, mask).mean() + return {C.LOSS:loss} + + def forward(self, words, seq_len, target): + return self._forward(words, seq_len, target) + + def predict(self, words, seq_len): + return self._forward(words, seq_len) class SeqLabeling(BaseModel): diff --git a/fastNLP/modules/encoder/embedding.py b/fastNLP/modules/encoder/embedding.py index 050a423a..0639959b 100644 --- a/fastNLP/modules/encoder/embedding.py +++ b/fastNLP/modules/encoder/embedding.py @@ -689,7 +689,7 @@ class BertEmbedding(ContextualEmbedding): outputs = self.model(words) outputs = torch.cat([*outputs], dim=-1) - return self.dropout(words) + return self.dropout(outputs) @property def requires_grad(self): diff --git a/fastNLP/modules/encoder/lstm.py b/fastNLP/modules/encoder/lstm.py index 5e599a65..695f56b8 100644 --- a/fastNLP/modules/encoder/lstm.py +++ b/fastNLP/modules/encoder/lstm.py @@ -10,10 +10,6 @@ import torch import torch.nn as nn import torch.nn.utils.rnn as rnn -from ..utils import initial_parameter -from torch import autograd - - class LSTM(nn.Module): """ 别名：:class:`fastNLP.modules.LSTM` :class:`fastNLP.modules.encoder.lstm.LSTM` diff --git a/reproduction/Biaffine_parser/cfg.cfg b/reproduction/legacy/Biaffine_parser/cfg.cfg similarity index 100% rename from reproduction/Biaffine_parser/cfg.cfg rename to reproduction/legacy/Biaffine_parser/cfg.cfg diff --git a/reproduction/Biaffine_parser/infer.py b/reproduction/legacy/Biaffine_parser/infer.py similarity index 100% rename from reproduction/Biaffine_parser/infer.py rename to reproduction/legacy/Biaffine_parser/infer.py diff --git a/reproduction/Biaffine_parser/main.py b/reproduction/legacy/Biaffine_parser/main.py similarity index 100% rename from reproduction/Biaffine_parser/main.py rename to reproduction/legacy/Biaffine_parser/main.py diff --git a/reproduction/Biaffine_parser/run.py b/reproduction/legacy/Biaffine_parser/run.py similarity index 100% rename from reproduction/Biaffine_parser/run.py rename to reproduction/legacy/Biaffine_parser/run.py diff --git a/reproduction/Biaffine_parser/util.py b/reproduction/legacy/Biaffine_parser/util.py similarity index 100% rename from reproduction/Biaffine_parser/util.py rename to reproduction/legacy/Biaffine_parser/util.py diff --git a/reproduction/Chinese_word_segmentation/__init__.py b/reproduction/legacy/Chinese_word_segmentation/__init__.py similarity index 100% rename from reproduction/Chinese_word_segmentation/__init__.py rename to reproduction/legacy/Chinese_word_segmentation/__init__.py diff --git a/reproduction/Chinese_word_segmentation/cws.cfg b/reproduction/legacy/Chinese_word_segmentation/cws.cfg similarity index 100% rename from reproduction/Chinese_word_segmentation/cws.cfg rename to reproduction/legacy/Chinese_word_segmentation/cws.cfg diff --git a/reproduction/Chinese_word_segmentation/cws_io/__init__.py b/reproduction/legacy/Chinese_word_segmentation/cws_io/__init__.py similarity index 100% rename from reproduction/Chinese_word_segmentation/cws_io/__init__.py rename to reproduction/legacy/Chinese_word_segmentation/cws_io/__init__.py diff --git a/reproduction/Chinese_word_segmentation/cws_io/cws_reader.py b/reproduction/legacy/Chinese_word_segmentation/cws_io/cws_reader.py similarity index 100% rename from reproduction/Chinese_word_segmentation/cws_io/cws_reader.py rename to reproduction/legacy/Chinese_word_segmentation/cws_io/cws_reader.py diff --git a/reproduction/Chinese_word_segmentation/models/__init__.py b/reproduction/legacy/Chinese_word_segmentation/models/__init__.py similarity index 100% rename from reproduction/Chinese_word_segmentation/models/__init__.py rename to reproduction/legacy/Chinese_word_segmentation/models/__init__.py diff --git a/reproduction/Chinese_word_segmentation/models/cws_model.py b/reproduction/legacy/Chinese_word_segmentation/models/cws_model.py similarity index 98% rename from reproduction/Chinese_word_segmentation/models/cws_model.py rename to reproduction/legacy/Chinese_word_segmentation/models/cws_model.py index b41ad87d..0d10d2e5 100644 --- a/reproduction/Chinese_word_segmentation/models/cws_model.py +++ b/reproduction/legacy/Chinese_word_segmentation/models/cws_model.py @@ -4,7 +4,7 @@ from torch import nn from fastNLP.models.base_model import BaseModel from fastNLP.modules.decoder.mlp import MLP -from reproduction.Chinese_word_segmentation.utils import seq_lens_to_mask +from reproduction.legacy.Chinese_word_segmentation.utils import seq_lens_to_mask class CWSBiLSTMEncoder(BaseModel): diff --git a/reproduction/Chinese_word_segmentation/models/cws_transformer.py b/reproduction/legacy/Chinese_word_segmentation/models/cws_transformer.py similarity index 97% rename from reproduction/Chinese_word_segmentation/models/cws_transformer.py rename to reproduction/legacy/Chinese_word_segmentation/models/cws_transformer.py index e8ae5ecc..ae8a5a7f 100644 --- a/reproduction/Chinese_word_segmentation/models/cws_transformer.py +++ b/reproduction/legacy/Chinese_word_segmentation/models/cws_transformer.py @@ -9,7 +9,7 @@ from torch import nn import torch # from fastNLP.modules.encoder.transformer import TransformerEncoder -from reproduction.Chinese_word_segmentation.models.transformer import TransformerEncoder +from reproduction.legacy.Chinese_word_segmentation.models import TransformerEncoder from fastNLP.modules.decoder.crf import ConditionalRandomField,seq_len_to_byte_mask from fastNLP.modules.decoder.crf import allowed_transitions @@ -79,7 +79,7 @@ class TransformerCWS(nn.Module): return {'pred': probs, 'seq_lens':seq_lens} -from reproduction.Chinese_word_segmentation.models.dilated_transformer import TransformerDilateEncoder +from reproduction.legacy.Chinese_word_segmentation.models import TransformerDilateEncoder class TransformerDilatedCWS(nn.Module): def __init__(self, vocab_num, embed_dim=100, bigram_vocab_num=None, bigram_embed_dim=100, num_bigram_per_char=None, diff --git a/reproduction/Chinese_word_segmentation/process/__init__.py b/reproduction/legacy/Chinese_word_segmentation/process/__init__.py similarity index 100% rename from reproduction/Chinese_word_segmentation/process/__init__.py rename to reproduction/legacy/Chinese_word_segmentation/process/__init__.py diff --git a/reproduction/Chinese_word_segmentation/process/cws_processor.py b/reproduction/legacy/Chinese_word_segmentation/process/cws_processor.py similarity index 99% rename from reproduction/Chinese_word_segmentation/process/cws_processor.py rename to reproduction/legacy/Chinese_word_segmentation/process/cws_processor.py index 614d9ef5..1f64bed2 100644 --- a/reproduction/Chinese_word_segmentation/process/cws_processor.py +++ b/reproduction/legacy/Chinese_word_segmentation/process/cws_processor.py @@ -4,7 +4,7 @@ import re from fastNLP.api.processor import Processor from fastNLP.core.dataset import DataSet from fastNLP.core.vocabulary import Vocabulary -from reproduction.Chinese_word_segmentation.process.span_converter import SpanConverter +from reproduction.legacy.Chinese_word_segmentation.process.span_converter import SpanConverter _SPECIAL_TAG_PATTERN = '<[a-zA-Z]+>' diff --git a/reproduction/Chinese_word_segmentation/process/span_converter.py b/reproduction/legacy/Chinese_word_segmentation/process/span_converter.py similarity index 100% rename from reproduction/Chinese_word_segmentation/process/span_converter.py rename to reproduction/legacy/Chinese_word_segmentation/process/span_converter.py diff --git a/reproduction/Chinese_word_segmentation/utils.py b/reproduction/legacy/Chinese_word_segmentation/utils.py similarity index 100% rename from reproduction/Chinese_word_segmentation/utils.py rename to reproduction/legacy/Chinese_word_segmentation/utils.py diff --git a/reproduction/LSTM+self_attention_sentiment_analysis/README.md b/reproduction/legacy/LSTM+self_attention_sentiment_analysis/README.md similarity index 100% rename from reproduction/LSTM+self_attention_sentiment_analysis/README.md rename to reproduction/legacy/LSTM+self_attention_sentiment_analysis/README.md diff --git a/reproduction/LSTM+self_attention_sentiment_analysis/Word2Idx.py b/reproduction/legacy/LSTM+self_attention_sentiment_analysis/Word2Idx.py similarity index 100% rename from reproduction/LSTM+self_attention_sentiment_analysis/Word2Idx.py rename to reproduction/legacy/LSTM+self_attention_sentiment_analysis/Word2Idx.py diff --git a/reproduction/LSTM+self_attention_sentiment_analysis/config.cfg b/reproduction/legacy/LSTM+self_attention_sentiment_analysis/config.cfg similarity index 100% rename from reproduction/LSTM+self_attention_sentiment_analysis/config.cfg rename to reproduction/legacy/LSTM+self_attention_sentiment_analysis/config.cfg diff --git a/reproduction/LSTM+self_attention_sentiment_analysis/dataloader.py b/reproduction/legacy/LSTM+self_attention_sentiment_analysis/dataloader.py similarity index 100% rename from reproduction/LSTM+self_attention_sentiment_analysis/dataloader.py rename to reproduction/legacy/LSTM+self_attention_sentiment_analysis/dataloader.py diff --git a/reproduction/LSTM+self_attention_sentiment_analysis/example.py b/reproduction/legacy/LSTM+self_attention_sentiment_analysis/example.py similarity index 100% rename from reproduction/LSTM+self_attention_sentiment_analysis/example.py rename to reproduction/legacy/LSTM+self_attention_sentiment_analysis/example.py diff --git a/reproduction/LSTM+self_attention_sentiment_analysis/main.py b/reproduction/legacy/LSTM+self_attention_sentiment_analysis/main.py similarity index 100% rename from reproduction/LSTM+self_attention_sentiment_analysis/main.py rename to reproduction/legacy/LSTM+self_attention_sentiment_analysis/main.py diff --git a/reproduction/LSTM+self_attention_sentiment_analysis/predict.py b/reproduction/legacy/LSTM+self_attention_sentiment_analysis/predict.py similarity index 100% rename from reproduction/LSTM+self_attention_sentiment_analysis/predict.py rename to reproduction/legacy/LSTM+self_attention_sentiment_analysis/predict.py diff --git a/reproduction/LSTM+self_attention_sentiment_analysis/prepare.py b/reproduction/legacy/LSTM+self_attention_sentiment_analysis/prepare.py similarity index 100% rename from reproduction/LSTM+self_attention_sentiment_analysis/prepare.py rename to reproduction/legacy/LSTM+self_attention_sentiment_analysis/prepare.py diff --git a/reproduction/POS_tagging/pos_processor.py b/reproduction/legacy/POS_tagging/pos_processor.py similarity index 100% rename from reproduction/POS_tagging/pos_processor.py rename to reproduction/legacy/POS_tagging/pos_processor.py diff --git a/reproduction/POS_tagging/pos_reader.py b/reproduction/legacy/POS_tagging/pos_reader.py similarity index 100% rename from reproduction/POS_tagging/pos_reader.py rename to reproduction/legacy/POS_tagging/pos_reader.py diff --git a/reproduction/POS_tagging/pos_tag.cfg b/reproduction/legacy/POS_tagging/pos_tag.cfg similarity index 100% rename from reproduction/POS_tagging/pos_tag.cfg rename to reproduction/legacy/POS_tagging/pos_tag.cfg diff --git a/reproduction/POS_tagging/train_pos_tag.py b/reproduction/legacy/POS_tagging/train_pos_tag.py similarity index 100% rename from reproduction/POS_tagging/train_pos_tag.py rename to reproduction/legacy/POS_tagging/train_pos_tag.py diff --git a/reproduction/POS_tagging/utils.py b/reproduction/legacy/POS_tagging/utils.py similarity index 100% rename from reproduction/POS_tagging/utils.py rename to reproduction/legacy/POS_tagging/utils.py diff --git a/reproduction/seqence_labelling/chinese_ner/data/ChineseNER.py b/reproduction/seqence_labelling/chinese_ner/data/ChineseNER.py new file mode 100644 index 00000000..cec5ab76 --- /dev/null +++ b/reproduction/seqence_labelling/chinese_ner/data/ChineseNER.py @@ -0,0 +1,115 @@ + + +from fastNLP.io.base_loader import DataSetLoader, DataBundle +from fastNLP.io import ConllLoader +from reproduction.seqence_labelling.ner.data.utils import iob2bioes, iob2 +from fastNLP import Const +from reproduction.utils import check_dataloader_paths +from fastNLP import Vocabulary + +class ChineseNERLoader(DataSetLoader): + """ + 读取中文命名实体数据集，包括PeopleDaily, MSRA-NER, Weibo。数据在这里可以找到https://github.com/OYE93/Chinese-NLP-Corpus/tree/master/NER + 请确保输入数据的格式如下, 共两列，第一列为字，第二列为标签，不同句子以空行隔开 + 我 O + 们 O + 变 O + 而 O + 以 O + 书 O + 会 O + ... + + """ + def __init__(self, encoding_type:str='bioes'): + """ + + :param str encoding_type: 支持bio和bioes格式 + """ + super().__init__() + self._loader = ConllLoader(headers=['raw_chars', 'target'], indexes=[0, 1]) + + assert encoding_type in ('bio', 'bioes') + + self._tag_converters = [iob2] + if encoding_type == 'bioes': + self._tag_converters.append(iob2bioes) + + def load(self, path:str): + dataset = self._loader.load(path) + def convert_tag_schema(tags): + for converter in self._tag_converters: + tags = converter(tags) + return tags + if self._tag_converters: + dataset.apply_field(convert_tag_schema, field_name=Const.TARGET, new_field_name=Const.TARGET) + return dataset + + def process(self, paths, bigrams=False, trigrams=False): + """ + + :param paths: + :param bool, bigrams: 是否包含生成bigram feature, [a, b, c, d] -> [ab, bc, cd, d] + :param bool, trigrams: 是否包含trigram feature，[a, b, c, d] -> [abc, bcd, cd, d] + :return: DataBundle + 包含以下的fields + raw_chars: List[str] + chars: List[int] + seq_len: int, 字的长度 + bigrams: List[int], optional + trigrams: List[int], optional + target: List[int] + """ + paths = check_dataloader_paths(paths) + data = DataBundle() + input_fields = [Const.CHAR_INPUT, Const.INPUT_LEN, Const.TARGET] + target_fields = [Const.TARGET, Const.INPUT_LEN] + + for name, path in paths.items(): + dataset = self.load(path) + if bigrams: + dataset.apply_field(lambda raw_chars: [c1+c2 for c1, c2 in zip(raw_chars, raw_chars[1:]+[''])], + field_name='raw_chars', new_field_name='bigrams') + + if trigrams: + dataset.apply_field(lambda raw_chars: [c1+c2+c3 for c1, c2, c3 in zip(raw_chars, + raw_chars[1:]+[''], + raw_chars[2:]+['']*2)], + field_name='raw_chars', new_field_name='trigrams') + data.datasets[name] = dataset + + char_vocab = Vocabulary().from_dataset(data.datasets['train'], field_name='raw_chars', + no_create_entry_dataset=[dataset for name, dataset in data.datasets.items() if name!='train']) + char_vocab.index_dataset(*data.datasets.values(), field_name='raw_chars', new_field_name=Const.CHAR_INPUT) + data.vocabs[Const.CHAR_INPUT] = char_vocab + + target_vocab = Vocabulary(unknown=None, padding=None).from_dataset(data.datasets['train'], field_name=Const.TARGET) + target_vocab.index_dataset(*data.datasets.values(), field_name=Const.TARGET) + data.vocabs[Const.TARGET] = target_vocab + + if bigrams: + bigram_vocab = Vocabulary().from_dataset(data.datasets['train'], field_name='bigrams', + no_create_entry_dataset=[dataset for name, dataset in + data.datasets.items() if name != 'train']) + bigram_vocab.index_dataset(*data.datasets.values(), field_name='bigrams', new_field_name='bigrams') + data.vocabs['bigrams'] = bigram_vocab + input_fields.append('bigrams') + + if trigrams: + trigram_vocab = Vocabulary().from_dataset(data.datasets['train'], field_name='trigrams', + no_create_entry_dataset=[dataset for name, dataset in + data.datasets.items() if name != 'train']) + trigram_vocab.index_dataset(*data.datasets.values(), field_name='trigrams', new_field_name='trigrams') + data.vocabs['trigrams'] = trigram_vocab + input_fields.append('trigrams') + + for name, dataset in data.datasets.items(): + dataset.add_seq_len(Const.CHAR_INPUT) + dataset.set_input(*input_fields) + dataset.set_target(*target_fields) + + return data + + + + diff --git a/reproduction/seqence_labelling/chinese_ner/data/__init__.py b/reproduction/seqence_labelling/chinese_ner/data/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/reproduction/seqence_labelling/chinese_ner/train_cn_ner.py b/reproduction/seqence_labelling/chinese_ner/train_cn_ner.py new file mode 100644 index 00000000..1993898b --- /dev/null +++ b/reproduction/seqence_labelling/chinese_ner/train_cn_ner.py @@ -0,0 +1,94 @@ + + + +from reproduction.seqence_labelling.chinese_ner.data.ChineseNER import ChineseNERLoader +from fastNLP.modules.encoder.embedding import StaticEmbedding + +from torch import nn +import torch +from fastNLP.modules import get_embeddings +from fastNLP.modules import LSTM +from fastNLP.modules import ConditionalRandomField +from fastNLP.modules import allowed_transitions +import torch.nn.functional as F +from fastNLP import seq_len_to_mask +from fastNLP.core.const import Const as C +from fastNLP import SpanFPreRecMetric, Trainer +from fastNLP import cache_results + +class CNBiLSTMCRFNER(nn.Module): + def __init__(self, char_embed, num_classes, bigram_embed=None, trigram_embed=None, num_layers=1, hidden_size=100, + dropout=0.5, target_vocab=None, encoding_type=None): + super().__init__() + + self.char_embed = get_embeddings(char_embed) + embed_size = self.char_embed.embedding_dim + if bigram_embed: + self.bigram_embed = get_embeddings(bigram_embed) + embed_size += self.bigram_embed.embedding_dim + if trigram_embed: + self.trigram_ebmbed = get_embeddings(trigram_embed) + embed_size += self.bigram_embed.embedding_dim + + if num_layers>1: + self.lstm = LSTM(embed_size, num_layers=num_layers, hidden_size=hidden_size//2, bidirectional=True, + batch_first=True, dropout=dropout) + else: + self.lstm = LSTM(embed_size, num_layers=num_layers, hidden_size=hidden_size//2, bidirectional=True, + batch_first=True) + + self.dropout = nn.Dropout(dropout) + self.fc = nn.Linear(hidden_size, num_classes) + + trans = None + if target_vocab is not None and encoding_type is not None: + trans = allowed_transitions(target_vocab.idx2word, encoding_type=encoding_type, include_start_end=True) + + self.crf = ConditionalRandomField(num_classes, include_start_end_trans=True, allowed_transitions=trans) + + def _forward(self, chars, bigrams=None, trigrams=None, seq_len=None, target=None): + chars = self.char_embed(chars) + if hasattr(self, 'bigram_embed'): + bigrams = self.bigram_embed(bigrams) + chars = torch.cat((chars, bigrams), dim=-1) + if hasattr(self, 'trigram_embed'): + trigrams = self.trigram_embed(trigrams) + chars = torch.cat((chars, trigrams), dim=-1) + feats, _ = self.lstm(chars, seq_len=seq_len) + feats = self.fc(feats) + feats = self.dropout(feats) + logits = F.log_softmax(feats, dim=-1) + mask = seq_len_to_mask(seq_len) + if target is None: + pred, _ = self.crf.viterbi_decode(logits, mask) + return {C.OUTPUT: pred} + else: + loss = self.crf(logits, target, mask).mean() + return {C.LOSS:loss} + + def forward(self, chars, target, bigrams=None, trigrams=None, seq_len=None): + return self._forward(chars, bigrams, trigrams, seq_len, target) + + def predict(self, chars, seq_len=None, bigrams=None, trigrams=None): + return self._forward(chars, bigrams, trigrams, seq_len) + +# data_bundle = pickle.load(open('caches/msra.pkl', 'rb')) +@cache_results('caches/msra.pkl', _refresh=False) +def get_data(): + data_bundle = ChineseNERLoader().process('/remote-home/hyan01/exps/fastNLP/others/data/MSRA-NER', bigrams=True) + char_embed = StaticEmbedding(data_bundle.vocabs['chars'], + model_dir_or_name='/remote-home/hyan01/exps/CWS/pretrain/vectors/1grams_t3_m50_corpus.txt') + bigram_embed = StaticEmbedding(data_bundle.vocabs['bigrams'], + model_dir_or_name='/remote-home/hyan01/exps/CWS/pretrain/vectors/2gram_t3_m50_merge.txt') + return data_bundle, char_embed, bigram_embed +data_bundle, char_embed, bigram_embed = get_data() +print(data_bundle) +# exit(0) +data_bundle.datasets['train'].set_input('target') +data_bundle.datasets['dev'].set_input('target') +model = CNBiLSTMCRFNER(char_embed, num_classes=len(data_bundle.vocabs['target']), bigram_embed=bigram_embed) + +Trainer(data_bundle.datasets['train'], model, batch_size=640, + metrics=SpanFPreRecMetric(data_bundle.vocabs['target'], encoding_type='bioes'), + num_workers=2, dev_data=data_bundle. datasets['dev'], device=3).train() + diff --git a/reproduction/seqence_labelling/ner/train_ontonote.py b/reproduction/seqence_labelling/ner/train_ontonote.py index 6548cb9f..33f015d8 100644 --- a/reproduction/seqence_labelling/ner/train_ontonote.py +++ b/reproduction/seqence_labelling/ner/train_ontonote.py @@ -12,54 +12,72 @@ from fastNLP import Const from torch.optim import SGD, Adam from torch.optim.lr_scheduler import LambdaLR from fastNLP import GradientClipCallback +from fastNLP.core.vocabulary import VocabularyOption from fastNLP.core.callback import FitlogCallback, LRScheduler -from reproduction.seqence_labelling.ner.model.swats import SWATS +from functools import partial +from torch import nn +from fastNLP import cache_results import fitlog fitlog.debug() +fitlog.set_log_dir('logs/') + +fitlog.add_hyper_in_file(__file__) +#######hyper +normalize = False +divide_std = True +lower = False +lr = 0.015 +dropout = 0.5 +batch_size = 20 +init_method = 'default' +job_embed = False +data_name = 'ontonote' +#######hyper + + +init_method = {'default': None, + 'xavier': partial(nn.init.xavier_normal_, gain=0.02), + 'normal': partial(nn.init.normal_, std=0.02) + }[init_method] + from reproduction.seqence_labelling.ner.data.OntoNoteLoader import OntoNoteNERDataLoader encoding_type = 'bioes' -data = OntoNoteNERDataLoader(encoding_type=encoding_type).process('/hdd/fudanNLP/fastNLP/others/data/v4/english', - lower=True) - -import joblib -raw_data = joblib.load('/hdd/fudanNLP/fastNLP/others/NER-with-LS/data/ontonotes_with_data.joblib') -def convert_to_ids(raw_words): - ids = [] - for word in raw_words: - id = raw_data['word_to_id'][word] - id = raw_data['id_to_emb_map'][id] - ids.append(id) - return ids -word_embed = raw_data['emb_matrix'] -for name, dataset in data.datasets.items(): - dataset.apply_field(convert_to_ids, field_name='raw_words', new_field_name=Const.INPUT) +@cache_results('caches/ontonotes.pkl') +def cache(): + data = OntoNoteNERDataLoader(encoding_type=encoding_type).process('../../../../others/data/v4/english', + lower=lower, + word_vocab_opt=VocabularyOption(min_freq=1)) + char_embed = CNNCharEmbedding(vocab=data.vocabs['cap_words'], embed_size=30, char_emb_size=30, filter_nums=[30], + kernel_sizes=[3]) + word_embed = StaticEmbedding(vocab=data.vocabs[Const.INPUT], + model_dir_or_name='/remote-home/hyan01/fastnlp_caches/glove.6B.100d/glove.6B.100d.txt', + requires_grad=True, + normalize=normalize, + init_method=init_method) + return data, char_embed, word_embed +data, char_embed, word_embed = cache() print(data) -char_embed = CNNCharEmbedding(vocab=data.vocabs['cap_words'], embed_size=30, char_emb_size=30, filter_nums=[30], - kernel_sizes=[3]) -# word_embed = StaticEmbedding(vocab=data.vocabs[Const.INPUT], -# model_dir_or_name='/hdd/fudanNLP/pretrain_vectors/glove.6B.100d.txt', -# requires_grad=True) model = CNNBiLSTMCRF(word_embed, char_embed, hidden_size=1200, num_layers=1, tag_vocab=data.vocabs[Const.TARGET], - encoding_type=encoding_type) + encoding_type=encoding_type, dropout=dropout) -callbacks = [GradientClipCallback(clip_value=5, clip_type='value'), - FitlogCallback(data.datasets['test'], verbose=1)] +callbacks = [ + GradientClipCallback(clip_value=5, clip_type='value'), + FitlogCallback(data.datasets['test'], verbose=1) + ] -optimizer = SGD(model.parameters(), lr=0.01, momentum=0.9) +optimizer = SGD(model.parameters(), lr=lr, momentum=0.9) scheduler = LRScheduler(LambdaLR(optimizer, lr_lambda=lambda epoch: 1 / (1 + 0.05 * epoch))) callbacks.append(scheduler) -# optimizer = SWATS(model.parameters(), verbose=True) -# optimizer = Adam(model.parameters(), lr=0.005) -trainer = Trainer(train_data=data.datasets['train'], model=model, optimizer=optimizer, sampler=BucketSampler(num_buckets=100), - device=0, dev_data=data.datasets['dev'], batch_size=10, +trainer = Trainer(train_data=data.datasets['dev'][:100], model=model, optimizer=optimizer, sampler=None, + device=0, dev_data=data.datasets['dev'][:100], batch_size=batch_size, metrics=SpanFPreRecMetric(tag_vocab=data.vocabs[Const.TARGET], encoding_type=encoding_type), callbacks=callbacks, num_workers=1, n_epochs=100) trainer.train() \ No newline at end of file diff --git a/reproduction/text_classification/data/IMDBLoader.py b/reproduction/text_classification/data/IMDBLoader.py index d57ee41b..94244431 100644 --- a/reproduction/text_classification/data/IMDBLoader.py +++ b/reproduction/text_classification/data/IMDBLoader.py @@ -10,7 +10,6 @@ from fastNLP import Const from functools import partial from reproduction.utils import check_dataloader_paths, get_tokenizer - class IMDBLoader(DataSetLoader): """ 读取IMDB数据集，DataSet包含以下fields: @@ -51,6 +50,7 @@ class IMDBLoader(DataSetLoader): datasets = {} info = DataBundle() + paths = check_dataloader_paths(paths) for name, path in paths.items(): dataset = self.load(path) datasets[name] = dataset diff --git a/reproduction/text_classification/model/lstm.py b/reproduction/text_classification/model/lstm.py index 388f3f1c..93b4d8a9 100644 --- a/reproduction/text_classification/model/lstm.py +++ b/reproduction/text_classification/model/lstm.py @@ -2,7 +2,7 @@ import torch import torch.nn as nn from fastNLP.core.const import Const as C from fastNLP.modules.encoder.lstm import LSTM -from fastNLP.modules import encoder +from fastNLP.modules import get_embeddings from fastNLP.modules.decoder.mlp import MLP @@ -13,14 +13,14 @@ class BiLSTMSentiment(nn.Module): num_layers=1, nfc=128): super(BiLSTMSentiment,self).__init__() - self.embed = encoder.Embedding(init_embed) + self.embed = get_embeddings(init_embed) self.lstm = LSTM(input_size=self.embed.embedding_dim, hidden_size=hidden_dim, num_layers=num_layers, bidirectional=True) - self.mlp = MLP(size_layer=[hidden_dim* 2, nfc, num_classes]) + self.mlp = MLP(size_layer=[hidden_dim*2, nfc, num_classes]) def forward(self, words): x_emb = self.embed(words) output, _ = self.lstm(x_emb) - output = self.mlp(output[:,-1,:]) + output = self.mlp(torch.max(output, dim=1)[0]) return {C.OUTPUT: output} def predict(self, words): diff --git a/reproduction/text_classification/train_bert.py b/reproduction/text_classification/train_bert.py index e69de29b..4db54958 100644 --- a/reproduction/text_classification/train_bert.py +++ b/reproduction/text_classification/train_bert.py @@ -0,0 +1,33 @@ +import sys +sys.path.append('../../') + +from reproduction.text_classification.data.IMDBLoader import IMDBLoader +from fastNLP.modules.encoder.embedding import BertEmbedding +from reproduction.text_classification.model.lstm import BiLSTMSentiment +from fastNLP import Trainer +from fastNLP import CrossEntropyLoss, AccuracyMetric +from fastNLP import cache_results +from fastNLP import Tester + +# 对返回结果进行缓存，下一次运行就会自动跳过预处理 +@cache_results('imdb.pkl') +def get_data(): + data_bundle = IMDBLoader().process('imdb/') + return data_bundle +data_bundle = get_data() + +print(data_bundle) + +# 删除超过512, 但由于英语中会把word进行word piece处理，所以截取的时候做一点的裕量 +data_bundle.datasets['train'].drop(lambda x:len(x['words'])>400) +data_bundle.datasets['dev'].drop(lambda x:len(x['words'])>400) +data_bundle.datasets['test'].drop(lambda x:len(x['words'])>400) +bert_embed = BertEmbedding(data_bundle.vocabs['words'], requires_grad=False, + model_dir_or_name="en-base") +model = BiLSTMSentiment(bert_embed, len(data_bundle.vocabs['target'])) + +Trainer(data_bundle.datasets['train'], model, optimizer=None, loss=CrossEntropyLoss(), device=0, + batch_size=10, dev_data=data_bundle.datasets['dev'], metrics=AccuracyMetric()).train() + +# 在测试集上测试一下效果 +Tester(data_bundle.datasets['test'], model, batch_size=32, metrics=AccuracyMetric()).test() \ No newline at end of file From 391793a9614b373fc8ee1659e256adc70b824a86 Mon Sep 17 00:00:00 2001 From: ChenXin Date: Fri, 12 Jul 2019 10:36:46 +0800 Subject: [PATCH 003/141] =?UTF-8?q?=E6=9C=80=E6=96=B0=E7=9A=84docs?= =?UTF-8?q?=E7=BB=93=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/fastNLP.core.batch.rst | 6 ++--- docs/source/fastNLP.core.callback.rst | 6 ++--- docs/source/fastNLP.core.const.rst | 6 ++--- docs/source/fastNLP.core.dataset.rst | 6 ++--- docs/source/fastNLP.core.field.rst | 6 ++--- docs/source/fastNLP.core.instance.rst | 6 ++--- docs/source/fastNLP.core.losses.rst | 6 ++--- docs/source/fastNLP.core.metrics.rst | 6 ++--- docs/source/fastNLP.core.optimizer.rst | 6 ++--- docs/source/fastNLP.core.rst | 9 ++++---- docs/source/fastNLP.core.sampler.rst | 6 ++--- docs/source/fastNLP.core.tester.rst | 6 ++--- docs/source/fastNLP.core.trainer.rst | 6 ++--- docs/source/fastNLP.core.utils.rst | 6 ++--- docs/source/fastNLP.core.vocabulary.rst | 6 ++--- .../fastNLP.embeddings.bert_embedding.rst | 7 ++++++ .../fastNLP.embeddings.char_embedding.rst | 7 ++++++ ...astNLP.embeddings.contextual_embedding.rst | 7 ++++++ .../fastNLP.embeddings.elmo_embedding.rst | 7 ++++++ docs/source/fastNLP.embeddings.embedding.rst | 7 ++++++ docs/source/fastNLP.embeddings.rst | 21 ++++++++++++++++++ .../fastNLP.embeddings.stack_embedding.rst | 7 ++++++ .../fastNLP.embeddings.static_embedding.rst | 7 ++++++ docs/source/fastNLP.embeddings.utils.rst | 7 ++++++ docs/source/fastNLP.io.base_loader.rst | 6 ++--- docs/source/fastNLP.io.config_io.rst | 7 ++++++ docs/source/fastNLP.io.data_loader.rst | 6 ++--- docs/source/fastNLP.io.dataset_loader.rst | 6 ++--- docs/source/fastNLP.io.embed_loader.rst | 6 ++--- docs/source/fastNLP.io.file_utils.rst | 7 ++++++ docs/source/fastNLP.io.model_io.rst | 6 ++--- docs/source/fastNLP.io.rst | 17 +++++++++----- docs/source/fastNLP.io.utils.rst | 7 ++++++ docs/source/fastNLP.models.base_model.rst | 7 ++++++ docs/source/fastNLP.models.bert.rst | 7 ++++++ .../source/fastNLP.models.biaffine_parser.rst | 6 ++--- ...fastNLP.models.cnn_text_classification.rst | 6 ++--- .../source/fastNLP.models.enas_controller.rst | 7 ++++++ docs/source/fastNLP.models.enas_model.rst | 7 ++++++ docs/source/fastNLP.models.enas_trainer.rst | 7 ++++++ docs/source/fastNLP.models.enas_utils.rst | 7 ++++++ docs/source/fastNLP.models.rst | 14 +++++++----- .../fastNLP.models.sequence_labeling.rst | 6 ++--- docs/source/fastNLP.models.snli.rst | 6 ++--- .../fastNLP.models.star_transformer.rst | 6 ++--- docs/source/fastNLP.modules.decoder.crf.rst | 7 ------ docs/source/fastNLP.modules.decoder.mlp.rst | 7 ------ docs/source/fastNLP.modules.decoder.rst | 16 +++----------- docs/source/fastNLP.modules.decoder.utils.rst | 7 ------ docs/source/fastNLP.modules.encoder.bert.rst | 7 ------ .../fastNLP.modules.encoder.char_encoder.rst | 7 ------ .../fastNLP.modules.encoder.conv_maxpool.rst | 7 ------ .../fastNLP.modules.encoder.embedding.rst | 7 ------ docs/source/fastNLP.modules.encoder.lstm.rst | 7 ------ docs/source/fastNLP.modules.encoder.rst | 22 +++---------------- ...stNLP.modules.encoder.star_transformer.rst | 7 ------ .../fastNLP.modules.encoder.transformer.rst | 7 ------ ...astNLP.modules.encoder.variational_rnn.rst | 7 ------ docs/source/fastNLP.modules.rst | 6 ++--- docs/source/fastNLP.rst | 18 +++++++-------- 60 files changed, 254 insertions(+), 209 deletions(-) create mode 100644 docs/source/fastNLP.embeddings.bert_embedding.rst create mode 100644 docs/source/fastNLP.embeddings.char_embedding.rst create mode 100644 docs/source/fastNLP.embeddings.contextual_embedding.rst create mode 100644 docs/source/fastNLP.embeddings.elmo_embedding.rst create mode 100644 docs/source/fastNLP.embeddings.embedding.rst create mode 100644 docs/source/fastNLP.embeddings.rst create mode 100644 docs/source/fastNLP.embeddings.stack_embedding.rst create mode 100644 docs/source/fastNLP.embeddings.static_embedding.rst create mode 100644 docs/source/fastNLP.embeddings.utils.rst create mode 100644 docs/source/fastNLP.io.config_io.rst create mode 100644 docs/source/fastNLP.io.file_utils.rst create mode 100644 docs/source/fastNLP.io.utils.rst create mode 100644 docs/source/fastNLP.models.base_model.rst create mode 100644 docs/source/fastNLP.models.bert.rst create mode 100644 docs/source/fastNLP.models.enas_controller.rst create mode 100644 docs/source/fastNLP.models.enas_model.rst create mode 100644 docs/source/fastNLP.models.enas_trainer.rst create mode 100644 docs/source/fastNLP.models.enas_utils.rst delete mode 100644 docs/source/fastNLP.modules.decoder.crf.rst delete mode 100644 docs/source/fastNLP.modules.decoder.mlp.rst delete mode 100644 docs/source/fastNLP.modules.decoder.utils.rst delete mode 100644 docs/source/fastNLP.modules.encoder.bert.rst delete mode 100644 docs/source/fastNLP.modules.encoder.char_encoder.rst delete mode 100644 docs/source/fastNLP.modules.encoder.conv_maxpool.rst delete mode 100644 docs/source/fastNLP.modules.encoder.embedding.rst delete mode 100644 docs/source/fastNLP.modules.encoder.lstm.rst delete mode 100644 docs/source/fastNLP.modules.encoder.star_transformer.rst delete mode 100644 docs/source/fastNLP.modules.encoder.transformer.rst delete mode 100644 docs/source/fastNLP.modules.encoder.variational_rnn.rst diff --git a/docs/source/fastNLP.core.batch.rst b/docs/source/fastNLP.core.batch.rst index 33a5b730..03008b52 100644 --- a/docs/source/fastNLP.core.batch.rst +++ b/docs/source/fastNLP.core.batch.rst @@ -2,6 +2,6 @@ fastNLP.core.batch ================== .. automodule:: fastNLP.core.batch - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.core.callback.rst b/docs/source/fastNLP.core.callback.rst index 31ec627b..74a7825d 100644 --- a/docs/source/fastNLP.core.callback.rst +++ b/docs/source/fastNLP.core.callback.rst @@ -2,6 +2,6 @@ fastNLP.core.callback ===================== .. automodule:: fastNLP.core.callback - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.core.const.rst b/docs/source/fastNLP.core.const.rst index c9e3bd97..330a8883 100644 --- a/docs/source/fastNLP.core.const.rst +++ b/docs/source/fastNLP.core.const.rst @@ -2,6 +2,6 @@ fastNLP.core.const ================== .. automodule:: fastNLP.core.const - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.core.dataset.rst b/docs/source/fastNLP.core.dataset.rst index b377cb0f..1ad94bb6 100644 --- a/docs/source/fastNLP.core.dataset.rst +++ b/docs/source/fastNLP.core.dataset.rst @@ -2,6 +2,6 @@ fastNLP.core.dataset ==================== .. automodule:: fastNLP.core.dataset - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.core.field.rst b/docs/source/fastNLP.core.field.rst index 7686e79a..7fc099c9 100644 --- a/docs/source/fastNLP.core.field.rst +++ b/docs/source/fastNLP.core.field.rst @@ -2,6 +2,6 @@ fastNLP.core.field ================== .. automodule:: fastNLP.core.field - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.core.instance.rst b/docs/source/fastNLP.core.instance.rst index 14393a91..6e496ac1 100644 --- a/docs/source/fastNLP.core.instance.rst +++ b/docs/source/fastNLP.core.instance.rst @@ -2,6 +2,6 @@ fastNLP.core.instance ===================== .. automodule:: fastNLP.core.instance - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.core.losses.rst b/docs/source/fastNLP.core.losses.rst index d2dd492b..8e63dfa1 100644 --- a/docs/source/fastNLP.core.losses.rst +++ b/docs/source/fastNLP.core.losses.rst @@ -2,6 +2,6 @@ fastNLP.core.losses =================== .. automodule:: fastNLP.core.losses - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.core.metrics.rst b/docs/source/fastNLP.core.metrics.rst index 69afff36..d3b87bb8 100644 --- a/docs/source/fastNLP.core.metrics.rst +++ b/docs/source/fastNLP.core.metrics.rst @@ -2,6 +2,6 @@ fastNLP.core.metrics ==================== .. automodule:: fastNLP.core.metrics - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.core.optimizer.rst b/docs/source/fastNLP.core.optimizer.rst index e2100d2e..c80be53f 100644 --- a/docs/source/fastNLP.core.optimizer.rst +++ b/docs/source/fastNLP.core.optimizer.rst @@ -2,6 +2,6 @@ fastNLP.core.optimizer ====================== .. automodule:: fastNLP.core.optimizer - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.core.rst b/docs/source/fastNLP.core.rst index 82c13e46..cacc6622 100644 --- a/docs/source/fastNLP.core.rst +++ b/docs/source/fastNLP.core.rst @@ -2,15 +2,15 @@ fastNLP.core ============ .. automodule:: fastNLP.core - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: 子模块 ---------- .. toctree:: - :titlesonly: + :maxdepth: 1 fastNLP.core.batch fastNLP.core.callback @@ -26,4 +26,3 @@ fastNLP.core fastNLP.core.trainer fastNLP.core.utils fastNLP.core.vocabulary - diff --git a/docs/source/fastNLP.core.sampler.rst b/docs/source/fastNLP.core.sampler.rst index 1810d59c..0110f0c0 100644 --- a/docs/source/fastNLP.core.sampler.rst +++ b/docs/source/fastNLP.core.sampler.rst @@ -2,6 +2,6 @@ fastNLP.core.sampler ==================== .. automodule:: fastNLP.core.sampler - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.core.tester.rst b/docs/source/fastNLP.core.tester.rst index a9e7e09f..4d71a27b 100644 --- a/docs/source/fastNLP.core.tester.rst +++ b/docs/source/fastNLP.core.tester.rst @@ -2,6 +2,6 @@ fastNLP.core.tester =================== .. automodule:: fastNLP.core.tester - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.core.trainer.rst b/docs/source/fastNLP.core.trainer.rst index 9e518d4b..60bf2d5b 100644 --- a/docs/source/fastNLP.core.trainer.rst +++ b/docs/source/fastNLP.core.trainer.rst @@ -2,6 +2,6 @@ fastNLP.core.trainer ==================== .. automodule:: fastNLP.core.trainer - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.core.utils.rst b/docs/source/fastNLP.core.utils.rst index fcd3f50c..3f80b4e8 100644 --- a/docs/source/fastNLP.core.utils.rst +++ b/docs/source/fastNLP.core.utils.rst @@ -2,6 +2,6 @@ fastNLP.core.utils ================== .. automodule:: fastNLP.core.utils - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.core.vocabulary.rst b/docs/source/fastNLP.core.vocabulary.rst index b3bf4bac..ba9598b9 100644 --- a/docs/source/fastNLP.core.vocabulary.rst +++ b/docs/source/fastNLP.core.vocabulary.rst @@ -2,6 +2,6 @@ fastNLP.core.vocabulary ======================= .. automodule:: fastNLP.core.vocabulary - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.embeddings.bert_embedding.rst b/docs/source/fastNLP.embeddings.bert_embedding.rst new file mode 100644 index 00000000..24ceff1c --- /dev/null +++ b/docs/source/fastNLP.embeddings.bert_embedding.rst @@ -0,0 +1,7 @@ +fastNLP.embeddings.bert\_embedding +================================== + +.. automodule:: fastNLP.embeddings.bert_embedding + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.embeddings.char_embedding.rst b/docs/source/fastNLP.embeddings.char_embedding.rst new file mode 100644 index 00000000..501089d8 --- /dev/null +++ b/docs/source/fastNLP.embeddings.char_embedding.rst @@ -0,0 +1,7 @@ +fastNLP.embeddings.char\_embedding +================================== + +.. automodule:: fastNLP.embeddings.char_embedding + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.embeddings.contextual_embedding.rst b/docs/source/fastNLP.embeddings.contextual_embedding.rst new file mode 100644 index 00000000..92e25c4d --- /dev/null +++ b/docs/source/fastNLP.embeddings.contextual_embedding.rst @@ -0,0 +1,7 @@ +fastNLP.embeddings.contextual\_embedding +======================================== + +.. automodule:: fastNLP.embeddings.contextual_embedding + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.embeddings.elmo_embedding.rst b/docs/source/fastNLP.embeddings.elmo_embedding.rst new file mode 100644 index 00000000..76669ee3 --- /dev/null +++ b/docs/source/fastNLP.embeddings.elmo_embedding.rst @@ -0,0 +1,7 @@ +fastNLP.embeddings.elmo\_embedding +================================== + +.. automodule:: fastNLP.embeddings.elmo_embedding + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.embeddings.embedding.rst b/docs/source/fastNLP.embeddings.embedding.rst new file mode 100644 index 00000000..5960d2cd --- /dev/null +++ b/docs/source/fastNLP.embeddings.embedding.rst @@ -0,0 +1,7 @@ +fastNLP.embeddings.embedding +============================ + +.. automodule:: fastNLP.embeddings.embedding + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.embeddings.rst b/docs/source/fastNLP.embeddings.rst new file mode 100644 index 00000000..25d05246 --- /dev/null +++ b/docs/source/fastNLP.embeddings.rst @@ -0,0 +1,21 @@ +fastNLP.embeddings +================== + +.. automodule:: fastNLP.embeddings + :members: + :undoc-members: + :show-inheritance: + +子模块 +---------- + +.. toctree:: + + fastNLP.embeddings.bert_embedding + fastNLP.embeddings.char_embedding + fastNLP.embeddings.contextual_embedding + fastNLP.embeddings.elmo_embedding + fastNLP.embeddings.embedding + fastNLP.embeddings.stack_embedding + fastNLP.embeddings.static_embedding + fastNLP.embeddings.utils diff --git a/docs/source/fastNLP.embeddings.stack_embedding.rst b/docs/source/fastNLP.embeddings.stack_embedding.rst new file mode 100644 index 00000000..4d2115f7 --- /dev/null +++ b/docs/source/fastNLP.embeddings.stack_embedding.rst @@ -0,0 +1,7 @@ +fastNLP.embeddings.stack\_embedding +=================================== + +.. automodule:: fastNLP.embeddings.stack_embedding + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.embeddings.static_embedding.rst b/docs/source/fastNLP.embeddings.static_embedding.rst new file mode 100644 index 00000000..e46de81a --- /dev/null +++ b/docs/source/fastNLP.embeddings.static_embedding.rst @@ -0,0 +1,7 @@ +fastNLP.embeddings.static\_embedding +==================================== + +.. automodule:: fastNLP.embeddings.static_embedding + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.embeddings.utils.rst b/docs/source/fastNLP.embeddings.utils.rst new file mode 100644 index 00000000..263bfbd6 --- /dev/null +++ b/docs/source/fastNLP.embeddings.utils.rst @@ -0,0 +1,7 @@ +fastNLP.embeddings.utils +======================== + +.. automodule:: fastNLP.embeddings.utils + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.io.base_loader.rst b/docs/source/fastNLP.io.base_loader.rst index c1f9ac14..057867f4 100644 --- a/docs/source/fastNLP.io.base_loader.rst +++ b/docs/source/fastNLP.io.base_loader.rst @@ -2,6 +2,6 @@ fastNLP.io.base\_loader ======================= .. automodule:: fastNLP.io.base_loader - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.io.config_io.rst b/docs/source/fastNLP.io.config_io.rst new file mode 100644 index 00000000..106d083d --- /dev/null +++ b/docs/source/fastNLP.io.config_io.rst @@ -0,0 +1,7 @@ +fastNLP.io.config\_io +===================== + +.. automodule:: fastNLP.io.config_io + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.io.data_loader.rst b/docs/source/fastNLP.io.data_loader.rst index 9261fa5c..8f990102 100644 --- a/docs/source/fastNLP.io.data_loader.rst +++ b/docs/source/fastNLP.io.data_loader.rst @@ -2,6 +2,6 @@ fastNLP.io.data\_loader ========================== .. automodule:: fastNLP.io.data_loader - :members: - :undoc-members: - :show-inheritance: \ No newline at end of file + :members: + :undoc-members: + :show-inheritance: \ No newline at end of file diff --git a/docs/source/fastNLP.io.dataset_loader.rst b/docs/source/fastNLP.io.dataset_loader.rst index d6663e59..e7990714 100644 --- a/docs/source/fastNLP.io.dataset_loader.rst +++ b/docs/source/fastNLP.io.dataset_loader.rst @@ -2,6 +2,6 @@ fastNLP.io.dataset\_loader ========================== .. automodule:: fastNLP.io.dataset_loader - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.io.embed_loader.rst b/docs/source/fastNLP.io.embed_loader.rst index 7a8e730c..69e1f7ff 100644 --- a/docs/source/fastNLP.io.embed_loader.rst +++ b/docs/source/fastNLP.io.embed_loader.rst @@ -2,6 +2,6 @@ fastNLP.io.embed\_loader ======================== .. automodule:: fastNLP.io.embed_loader - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.io.file_utils.rst b/docs/source/fastNLP.io.file_utils.rst new file mode 100644 index 00000000..4a46e889 --- /dev/null +++ b/docs/source/fastNLP.io.file_utils.rst @@ -0,0 +1,7 @@ +fastNLP.io.file_utils +===================== + +.. automodule:: fastNLP.io.file_utils + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.io.model_io.rst b/docs/source/fastNLP.io.model_io.rst index 50d4c25a..537ce752 100644 --- a/docs/source/fastNLP.io.model_io.rst +++ b/docs/source/fastNLP.io.model_io.rst @@ -2,6 +2,6 @@ fastNLP.io.model\_io ==================== .. automodule:: fastNLP.io.model_io - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.io.rst b/docs/source/fastNLP.io.rst index 33afbbee..da9dbcfe 100644 --- a/docs/source/fastNLP.io.rst +++ b/docs/source/fastNLP.io.rst @@ -2,19 +2,26 @@ fastNLP.io ========== .. automodule:: fastNLP.io - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: 子模块 ---------- .. toctree:: - :titlesonly: fastNLP.io.data_loader + +Submodules +---------- + +.. toctree:: + fastNLP.io.base_loader + fastNLP.io.config_io fastNLP.io.dataset_loader fastNLP.io.embed_loader + fastNLP.io.file_utils fastNLP.io.model_io - + fastNLP.io.utils diff --git a/docs/source/fastNLP.io.utils.rst b/docs/source/fastNLP.io.utils.rst new file mode 100644 index 00000000..0b3f3938 --- /dev/null +++ b/docs/source/fastNLP.io.utils.rst @@ -0,0 +1,7 @@ +fastNLP.io.utils +================ + +.. automodule:: fastNLP.io.utils + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.models.base_model.rst b/docs/source/fastNLP.models.base_model.rst new file mode 100644 index 00000000..fab1e6bf --- /dev/null +++ b/docs/source/fastNLP.models.base_model.rst @@ -0,0 +1,7 @@ +fastNLP.models.base\_model +========================== + +.. automodule:: fastNLP.models.base_model + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.models.bert.rst b/docs/source/fastNLP.models.bert.rst new file mode 100644 index 00000000..3b9482eb --- /dev/null +++ b/docs/source/fastNLP.models.bert.rst @@ -0,0 +1,7 @@ +fastNLP.models.bert +=================== + +.. automodule:: fastNLP.models.bert + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.models.biaffine_parser.rst b/docs/source/fastNLP.models.biaffine_parser.rst index a3dd1836..f19504e8 100644 --- a/docs/source/fastNLP.models.biaffine_parser.rst +++ b/docs/source/fastNLP.models.biaffine_parser.rst @@ -2,6 +2,6 @@ fastNLP.models.biaffine\_parser =============================== .. automodule:: fastNLP.models.biaffine_parser - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.models.cnn_text_classification.rst b/docs/source/fastNLP.models.cnn_text_classification.rst index a935d0bf..eacf6916 100644 --- a/docs/source/fastNLP.models.cnn_text_classification.rst +++ b/docs/source/fastNLP.models.cnn_text_classification.rst @@ -2,6 +2,6 @@ fastNLP.models.cnn\_text\_classification ======================================== .. automodule:: fastNLP.models.cnn_text_classification - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.models.enas_controller.rst b/docs/source/fastNLP.models.enas_controller.rst new file mode 100644 index 00000000..1ac97fac --- /dev/null +++ b/docs/source/fastNLP.models.enas_controller.rst @@ -0,0 +1,7 @@ +fastNLP.models.enas\_controller module +====================================== + +.. automodule:: fastNLP.models.enas_controller + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.models.enas_model.rst b/docs/source/fastNLP.models.enas_model.rst new file mode 100644 index 00000000..92dbb810 --- /dev/null +++ b/docs/source/fastNLP.models.enas_model.rst @@ -0,0 +1,7 @@ +fastNLP.models.enas\_model +========================== + +.. automodule:: fastNLP.models.enas_model + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.models.enas_trainer.rst b/docs/source/fastNLP.models.enas_trainer.rst new file mode 100644 index 00000000..eea79599 --- /dev/null +++ b/docs/source/fastNLP.models.enas_trainer.rst @@ -0,0 +1,7 @@ +fastNLP.models.enas\_trainer +============================ + +.. automodule:: fastNLP.models.enas_trainer + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.models.enas_utils.rst b/docs/source/fastNLP.models.enas_utils.rst new file mode 100644 index 00000000..c5e45b5f --- /dev/null +++ b/docs/source/fastNLP.models.enas_utils.rst @@ -0,0 +1,7 @@ +fastNLP.models.enas\_utils +========================== + +.. automodule:: fastNLP.models.enas_utils + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.models.rst b/docs/source/fastNLP.models.rst index 5858ebcd..5a237569 100644 --- a/docs/source/fastNLP.models.rst +++ b/docs/source/fastNLP.models.rst @@ -2,19 +2,23 @@ fastNLP.models ============== .. automodule:: fastNLP.models - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: 子模块 ---------- .. toctree:: - :titlesonly: + fastNLP.models.base_model + fastNLP.models.bert fastNLP.models.biaffine_parser fastNLP.models.cnn_text_classification + fastNLP.models.enas_controller + fastNLP.models.enas_model + fastNLP.models.enas_trainer + fastNLP.models.enas_utils fastNLP.models.sequence_labeling fastNLP.models.snli fastNLP.models.star_transformer - diff --git a/docs/source/fastNLP.models.sequence_labeling.rst b/docs/source/fastNLP.models.sequence_labeling.rst index 6d569fe1..85e28f06 100644 --- a/docs/source/fastNLP.models.sequence_labeling.rst +++ b/docs/source/fastNLP.models.sequence_labeling.rst @@ -2,6 +2,6 @@ fastNLP.models.sequence\_labeling ================================= .. automodule:: fastNLP.models.sequence_labeling - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.models.snli.rst b/docs/source/fastNLP.models.snli.rst index 24c2cc53..3b9b555c 100644 --- a/docs/source/fastNLP.models.snli.rst +++ b/docs/source/fastNLP.models.snli.rst @@ -2,6 +2,6 @@ fastNLP.models.snli =================== .. automodule:: fastNLP.models.snli - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.models.star_transformer.rst b/docs/source/fastNLP.models.star_transformer.rst index c93fb8cd..69d5c5b2 100644 --- a/docs/source/fastNLP.models.star_transformer.rst +++ b/docs/source/fastNLP.models.star_transformer.rst @@ -2,6 +2,6 @@ fastNLP.models.star\_transformer ================================ .. automodule:: fastNLP.models.star_transformer - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.modules.decoder.crf.rst b/docs/source/fastNLP.modules.decoder.crf.rst deleted file mode 100644 index 6d5b0d5b..00000000 --- a/docs/source/fastNLP.modules.decoder.crf.rst +++ /dev/null @@ -1,7 +0,0 @@ -fastNLP.modules.decoder.CRF -=========================== - -.. automodule:: fastNLP.modules.decoder.crf - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/fastNLP.modules.decoder.mlp.rst b/docs/source/fastNLP.modules.decoder.mlp.rst deleted file mode 100644 index 7d661ebf..00000000 --- a/docs/source/fastNLP.modules.decoder.mlp.rst +++ /dev/null @@ -1,7 +0,0 @@ -fastNLP.modules.decoder.MLP -=========================== - -.. automodule:: fastNLP.modules.decoder.mlp - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/fastNLP.modules.decoder.rst b/docs/source/fastNLP.modules.decoder.rst index e42a9f39..ecc2adbd 100644 --- a/docs/source/fastNLP.modules.decoder.rst +++ b/docs/source/fastNLP.modules.decoder.rst @@ -2,17 +2,7 @@ fastNLP.modules.decoder ======================= .. automodule:: fastNLP.modules.decoder - :members: - :undoc-members: - :show-inheritance: - -子模块 ----------- - -.. toctree:: - :titlesonly: - - fastNLP.modules.decoder.crf - fastNLP.modules.decoder.mlp - fastNLP.modules.decoder.utils + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.modules.decoder.utils.rst b/docs/source/fastNLP.modules.decoder.utils.rst deleted file mode 100644 index da979d99..00000000 --- a/docs/source/fastNLP.modules.decoder.utils.rst +++ /dev/null @@ -1,7 +0,0 @@ -fastNLP.modules.decoder.utils -============================= - -.. automodule:: fastNLP.modules.decoder.utils - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/fastNLP.modules.encoder.bert.rst b/docs/source/fastNLP.modules.encoder.bert.rst deleted file mode 100644 index 66bd0bbd..00000000 --- a/docs/source/fastNLP.modules.encoder.bert.rst +++ /dev/null @@ -1,7 +0,0 @@ -fastNLP.modules.encoder.bert -============================ - -.. automodule:: fastNLP.modules.encoder.bert - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/fastNLP.modules.encoder.char_encoder.rst b/docs/source/fastNLP.modules.encoder.char_encoder.rst deleted file mode 100644 index 61ea3340..00000000 --- a/docs/source/fastNLP.modules.encoder.char_encoder.rst +++ /dev/null @@ -1,7 +0,0 @@ -fastNLP.modules.encoder.char\_encoder -===================================== - -.. automodule:: fastNLP.modules.encoder.char_encoder - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/fastNLP.modules.encoder.conv_maxpool.rst b/docs/source/fastNLP.modules.encoder.conv_maxpool.rst deleted file mode 100644 index 7058a723..00000000 --- a/docs/source/fastNLP.modules.encoder.conv_maxpool.rst +++ /dev/null @@ -1,7 +0,0 @@ -fastNLP.modules.encoder.conv\_maxpool -===================================== - -.. automodule:: fastNLP.modules.encoder.conv_maxpool - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/fastNLP.modules.encoder.embedding.rst b/docs/source/fastNLP.modules.encoder.embedding.rst deleted file mode 100644 index 4427b3bf..00000000 --- a/docs/source/fastNLP.modules.encoder.embedding.rst +++ /dev/null @@ -1,7 +0,0 @@ -fastNLP.modules.encoder.embedding -================================= - -.. automodule:: fastNLP.modules.encoder.embedding - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/fastNLP.modules.encoder.lstm.rst b/docs/source/fastNLP.modules.encoder.lstm.rst deleted file mode 100644 index f9cbea88..00000000 --- a/docs/source/fastNLP.modules.encoder.lstm.rst +++ /dev/null @@ -1,7 +0,0 @@ -fastNLP.modules.encoder.lstm -============================ - -.. automodule:: fastNLP.modules.encoder.lstm - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/fastNLP.modules.encoder.rst b/docs/source/fastNLP.modules.encoder.rst index b15232fa..0562f12d 100644 --- a/docs/source/fastNLP.modules.encoder.rst +++ b/docs/source/fastNLP.modules.encoder.rst @@ -2,22 +2,6 @@ fastNLP.modules.encoder ======================= .. automodule:: fastNLP.modules.encoder - :members: - :undoc-members: - :show-inheritance: - -子模块 ----------- - -.. toctree:: - :titlesonly: - - fastNLP.modules.encoder.bert - fastNLP.modules.encoder.char_encoder - fastNLP.modules.encoder.conv_maxpool - fastNLP.modules.encoder.embedding - fastNLP.modules.encoder.lstm - fastNLP.modules.encoder.star_transformer - fastNLP.modules.encoder.transformer - fastNLP.modules.encoder.variational_rnn - + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.modules.encoder.star_transformer.rst b/docs/source/fastNLP.modules.encoder.star_transformer.rst deleted file mode 100644 index 0c406782..00000000 --- a/docs/source/fastNLP.modules.encoder.star_transformer.rst +++ /dev/null @@ -1,7 +0,0 @@ -fastNLP.modules.encoder.star\_transformer -========================================= - -.. automodule:: fastNLP.modules.encoder.star_transformer - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/fastNLP.modules.encoder.transformer.rst b/docs/source/fastNLP.modules.encoder.transformer.rst deleted file mode 100644 index 6a40c597..00000000 --- a/docs/source/fastNLP.modules.encoder.transformer.rst +++ /dev/null @@ -1,7 +0,0 @@ -fastNLP.modules.encoder.transformer -=================================== - -.. automodule:: fastNLP.modules.encoder.transformer - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/fastNLP.modules.encoder.variational_rnn.rst b/docs/source/fastNLP.modules.encoder.variational_rnn.rst deleted file mode 100644 index 348fb3d8..00000000 --- a/docs/source/fastNLP.modules.encoder.variational_rnn.rst +++ /dev/null @@ -1,7 +0,0 @@ -fastNLP.modules.encoder.variational\_rnn -======================================== - -.. automodule:: fastNLP.modules.encoder.variational_rnn - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/fastNLP.modules.rst b/docs/source/fastNLP.modules.rst index 7f75cfdc..3160fe99 100644 --- a/docs/source/fastNLP.modules.rst +++ b/docs/source/fastNLP.modules.rst @@ -2,9 +2,9 @@ fastNLP.modules =============== .. automodule:: fastNLP.modules - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: 子模块 ----------- diff --git a/docs/source/fastNLP.rst b/docs/source/fastNLP.rst index f0c3d41c..076d9df0 100644 --- a/docs/source/fastNLP.rst +++ b/docs/source/fastNLP.rst @@ -2,19 +2,17 @@ API 文档 =============== .. automodule:: fastNLP - :members: - :undoc-members: - :show-inheritance: + :members: + :undoc-members: + :show-inheritance: 内部模块 ----------- .. toctree:: - :titlesonly: - :maxdepth: 3 - - fastNLP.core - fastNLP.io - fastNLP.modules - fastNLP.models + fastNLP.core + fastNLP.embeddings + fastNLP.io + fastNLP.models + fastNLP.modules From c99f02aa3b9abf1d2ccd1db3bccbea0407cfb1ec Mon Sep 17 00:00:00 2001 From: ChenXin Date: Fri, 12 Jul 2019 10:46:29 +0800 Subject: [PATCH 004/141] =?UTF-8?q?API=E6=96=87=E6=A1=A3=E5=85=A5=E5=8F=A3?= =?UTF-8?q?=E7=9A=84=E4=BB=8B=E7=BB=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/__init__.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/fastNLP/__init__.py b/fastNLP/__init__.py index 0eaa5d81..ea2cd13a 100644 --- a/fastNLP/__init__.py +++ b/fastNLP/__init__.py @@ -1,11 +1,12 @@ """ -fastNLP 由 :mod:`~fastNLP.core` 、 :mod:`~fastNLP.io` 、:mod:`~fastNLP.modules`、:mod:`~fastNLP.models` -等子模块组成，你可以点进去查看每个模块的文档。 +fastNLP 由 :mod:`~fastNLP.core` 、 :mod:`~fastNLP.io` 、:mod:`~fastNLP.embeddings` 、 :mod:`~fastNLP.modules`、 +:mod:`~fastNLP.models` 等子模块组成，你可以查看每个模块的文档。 -- :mod:`~fastNLP.core` 是fastNLP 的核心模块，包括 DataSet、 Trainer、 Tester 等组件。详见文档 :doc:`/fastNLP.core` -- :mod:`~fastNLP.io` 是实现输入输出的模块，包括了数据集的读取，模型的存取等功能。详见文档 :doc:`/fastNLP.io` -- :mod:`~fastNLP.modules` 包含了用于搭建神经网络模型的诸多组件，可以帮助用户快速搭建自己所需的网络。详见文档 :doc:`/fastNLP.modules` -- :mod:`~fastNLP.models` 包含了一些使用 fastNLP 实现的完整网络模型，包括 :class:`~fastNLP.models.CNNText` 、 :class:`~fastNLP.models.SeqLabeling` 等常见模型。详见文档 :doc:`/fastNLP.models` +- :mod:`~fastNLP.core` 是fastNLP 的核心模块，包括 DataSet、 Trainer、 Tester 等组件。详见文档 :doc:`fastNLP.core` +- :mod:`~fastNLP.io` 是实现输入输出的模块，包括了数据集的读取，模型的存取等功能。详见文档 :doc:`fastNLP.io` +- :mod:`~fastNLP.embeddings` 提供用于构建复杂网络模型所需的各种embedding。详见文档 :doc:`fastNLP.embeddings` +- :mod:`~fastNLP.modules` 包含了用于搭建神经网络模型的诸多组件，可以帮助用户快速搭建自己所需的网络。详见文档 :doc:`fastNLP.modules` +- :mod:`~fastNLP.models` 包含了一些使用 fastNLP 实现的完整网络模型，包括 :class:`~fastNLP.models.CNNText` 、 :class:`~fastNLP.models.SeqLabeling` 等常见模型。详见文档 :doc:`fastNLP.models` fastNLP 中最常用的组件可以直接从 fastNLP 包中 import ，他们的文档如下： """ @@ -61,4 +62,5 @@ __version__ = '0.4.5' from .core import * from . import models from . import modules +from . import embeddings from .io import data_loader From d6ae241bbb51df3c8636331f4fc4607741cd3dd7 Mon Sep 17 00:00:00 2001 From: ChenXin Date: Fri, 12 Jul 2019 10:49:18 +0800 Subject: [PATCH 005/141] =?UTF-8?q?decoder=E9=83=A8=E5=88=86=E7=9A=84?= =?UTF-8?q?=E5=88=AB=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/modules/decoder/crf.py | 52 ++++++++++++++++---------------- fastNLP/modules/decoder/mlp.py | 10 +++--- fastNLP/modules/decoder/utils.py | 10 +++--- 3 files changed, 36 insertions(+), 36 deletions(-) diff --git a/fastNLP/modules/decoder/crf.py b/fastNLP/modules/decoder/crf.py index c0717d6f..7c496868 100644 --- a/fastNLP/modules/decoder/crf.py +++ b/fastNLP/modules/decoder/crf.py @@ -11,7 +11,7 @@ from ..utils import initial_parameter def allowed_transitions(id2target, encoding_type='bio', include_start_end=False): """ - 别名：:class:`fastNLP.modules.allowed_transitions` :class:`fastNLP.modules.decoder.crf.allowed_transitions` + 别名：:class:`fastNLP.modules.allowed_transitions` :class:`fastNLP.modules.decoder.allowed_transitions` 给定一个id到label的映射表，返回所有可以跳转的(from_tag_id, to_tag_id)列表。 @@ -31,7 +31,7 @@ def allowed_transitions(id2target, encoding_type='bio', include_start_end=False) id_label_lst = list(id2target.items()) if include_start_end: id_label_lst += [(start_idx, 'start'), (end_idx, 'end')] - + def split_tag_label(from_label): from_label = from_label.lower() if from_label in ['start', 'end']: @@ -41,7 +41,7 @@ def allowed_transitions(id2target, encoding_type='bio', include_start_end=False) from_tag = from_label[:1] from_label = from_label[2:] return from_tag, from_label - + for from_id, from_label in id_label_lst: if from_label in ['', '']: continue @@ -93,7 +93,7 @@ def _is_transition_allowed(encoding_type, from_tag, from_label, to_tag, to_label return to_tag in ['end', 'b', 'o'] else: raise ValueError("Unexpect tag {}. Expect only 'B', 'I', 'O'.".format(from_tag)) - + elif encoding_type == 'bmes': """ 第一行是to_tag, 第一列是from_tag，y任意条件下可转，-只有在label相同时可转，n不可转 @@ -151,7 +151,7 @@ def _is_transition_allowed(encoding_type, from_tag, from_label, to_tag, to_label class ConditionalRandomField(nn.Module): """ - 别名：:class:`fastNLP.modules.ConditionalRandomField` :class:`fastNLP.modules.decoder.crf.ConditionalRandomField` + 别名：:class:`fastNLP.modules.ConditionalRandomField` :class:`fastNLP.modules.decoder.ConditionalRandomField` 条件随机场。 提供forward()以及viterbi_decode()两个方法，分别用于训练与inference。 @@ -163,21 +163,21 @@ class ConditionalRandomField(nn.Module): allowed_transitions()函数得到；如果为None，则所有跃迁均为合法 :param str initial_method: 初始化方法。见initial_parameter """ - + def __init__(self, num_tags, include_start_end_trans=False, allowed_transitions=None, initial_method=None): - + super(ConditionalRandomField, self).__init__() - + self.include_start_end_trans = include_start_end_trans self.num_tags = num_tags - + # the meaning of entry in this matrix is (from_tag_id, to_tag_id) score self.trans_m = nn.Parameter(torch.randn(num_tags, num_tags)) if self.include_start_end_trans: self.start_scores = nn.Parameter(torch.randn(num_tags)) self.end_scores = nn.Parameter(torch.randn(num_tags)) - + if allowed_transitions is None: constrain = torch.zeros(num_tags + 2, num_tags + 2) else: @@ -185,9 +185,9 @@ class ConditionalRandomField(nn.Module): for from_tag_id, to_tag_id in allowed_transitions: constrain[from_tag_id, to_tag_id] = 0 self._constrain = nn.Parameter(constrain, requires_grad=False) - + initial_parameter(self, initial_method) - + def _normalizer_likelihood(self, logits, mask): """Computes the (batch_size,) denominator term for the log-likelihood, which is the sum of the likelihoods across all possible state sequences. @@ -200,21 +200,21 @@ class ConditionalRandomField(nn.Module): alpha = logits[0] if self.include_start_end_trans: alpha = alpha + self.start_scores.view(1, -1) - + flip_mask = mask.eq(0) - + for i in range(1, seq_len): emit_score = logits[i].view(batch_size, 1, n_tags) trans_score = self.trans_m.view(1, n_tags, n_tags) tmp = alpha.view(batch_size, n_tags, 1) + emit_score + trans_score alpha = torch.logsumexp(tmp, 1).masked_fill(flip_mask[i].view(batch_size, 1), 0) + \ alpha.masked_fill(mask[i].byte().view(batch_size, 1), 0) - + if self.include_start_end_trans: alpha = alpha + self.end_scores.view(1, -1) - + return torch.logsumexp(alpha, 1) - + def _gold_score(self, logits, tags, mask): """ Compute the score for the gold path. @@ -226,7 +226,7 @@ class ConditionalRandomField(nn.Module): seq_len, batch_size, _ = logits.size() batch_idx = torch.arange(batch_size, dtype=torch.long, device=logits.device) seq_idx = torch.arange(seq_len, dtype=torch.long, device=logits.device) - + # trans_socre [L-1, B] mask = mask.byte() flip_mask = mask.eq(0) @@ -243,7 +243,7 @@ class ConditionalRandomField(nn.Module): score = score + st_scores + ed_scores # return [B,] return score - + def forward(self, feats, tags, mask): """ 用于计算CRF的前向loss，返回值为一个batch_size的FloatTensor，可能需要mean()求得loss。 @@ -258,9 +258,9 @@ class ConditionalRandomField(nn.Module): mask = mask.transpose(0, 1).float() all_path_score = self._normalizer_likelihood(feats, mask) gold_path_score = self._gold_score(feats, tags, mask) - + return all_path_score - gold_path_score - + def viterbi_decode(self, logits, mask, unpad=False): """给定一个特征矩阵以及转移分数矩阵，计算出最佳的路径以及对应的分数 @@ -277,7 +277,7 @@ class ConditionalRandomField(nn.Module): batch_size, seq_len, n_tags = logits.size() logits = logits.transpose(0, 1).data # L, B, H mask = mask.transpose(0, 1).data.byte() # L, B - + # dp vpath = logits.new_zeros((seq_len, batch_size, n_tags), dtype=torch.long) vscore = logits[0] @@ -286,7 +286,7 @@ class ConditionalRandomField(nn.Module): if self.include_start_end_trans: transitions[n_tags, :n_tags] += self.start_scores.data transitions[:n_tags, n_tags + 1] += self.end_scores.data - + vscore += transitions[n_tags, :n_tags] trans_score = transitions[:n_tags, :n_tags].view(1, n_tags, n_tags).data for i in range(1, seq_len): @@ -297,17 +297,17 @@ class ConditionalRandomField(nn.Module): vpath[i] = best_dst vscore = best_score.masked_fill(mask[i].eq(0).view(batch_size, 1), 0) + \ vscore.masked_fill(mask[i].view(batch_size, 1), 0) - + if self.include_start_end_trans: vscore += transitions[:n_tags, n_tags + 1].view(1, -1) - + # backtrace batch_idx = torch.arange(batch_size, dtype=torch.long, device=logits.device) seq_idx = torch.arange(seq_len, dtype=torch.long, device=logits.device) lens = (mask.long().sum(0) - 1) # idxes [L, B], batched idx from seq_len-1 to 0 idxes = (lens.view(1, -1) - seq_idx.view(-1, 1)) % seq_len - + ans = logits.new_empty((seq_len, batch_size), dtype=torch.long) ans_score, last_tags = vscore.max(1) ans[idxes[0], batch_idx] = last_tags diff --git a/fastNLP/modules/decoder/mlp.py b/fastNLP/modules/decoder/mlp.py index 418b3a77..9d9d80f2 100644 --- a/fastNLP/modules/decoder/mlp.py +++ b/fastNLP/modules/decoder/mlp.py @@ -10,7 +10,7 @@ from ..utils import initial_parameter class MLP(nn.Module): """ - 别名：:class:`fastNLP.modules.MLP` :class:`fastNLP.modules.decoder.mlp.MLP` + 别名：:class:`fastNLP.modules.MLP` :class:`fastNLP.modules.decoder.MLP` 多层感知器 @@ -40,7 +40,7 @@ class MLP(nn.Module): >>> print(x) >>> print(y) """ - + def __init__(self, size_layer, activation='relu', output_activation=None, initial_method=None, dropout=0.0): super(MLP, self).__init__() self.hiddens = nn.ModuleList() @@ -51,9 +51,9 @@ class MLP(nn.Module): self.output = nn.Linear(size_layer[i - 1], size_layer[i]) else: self.hiddens.append(nn.Linear(size_layer[i - 1], size_layer[i])) - + self.dropout = nn.Dropout(p=dropout) - + actives = { 'relu': nn.ReLU(), 'tanh': nn.Tanh(), @@ -82,7 +82,7 @@ class MLP(nn.Module): else: raise ValueError("should set activation correctly: {}".format(activation)) initial_parameter(self, initial_method) - + def forward(self, x): """ :param torch.Tensor x: MLP接受的输入 diff --git a/fastNLP/modules/decoder/utils.py b/fastNLP/modules/decoder/utils.py index 249f3ff6..9e773336 100644 --- a/fastNLP/modules/decoder/utils.py +++ b/fastNLP/modules/decoder/utils.py @@ -6,7 +6,7 @@ import torch def viterbi_decode(logits, transitions, mask=None, unpad=False): r""" - 别名：:class:`fastNLP.modules.viterbi_decode` :class:`fastNLP.modules.decoder.utils.viterbi_decode` + 别名：:class:`fastNLP.modules.viterbi_decode` :class:`fastNLP.modules.decoder.viterbi_decode` 给定一个特征矩阵以及转移分数矩阵，计算出最佳的路径以及对应的分数 @@ -30,11 +30,11 @@ def viterbi_decode(logits, transitions, mask=None, unpad=False): mask = mask.transpose(0, 1).data.byte() # L, B else: mask = logits.new_ones((seq_len, batch_size), dtype=torch.uint8) - + # dp vpath = logits.new_zeros((seq_len, batch_size, n_tags), dtype=torch.long) vscore = logits[0] - + trans_score = transitions.view(1, n_tags, n_tags).data for i in range(1, seq_len): prev_score = vscore.view(batch_size, n_tags, 1) @@ -44,14 +44,14 @@ def viterbi_decode(logits, transitions, mask=None, unpad=False): vpath[i] = best_dst vscore = best_score.masked_fill(mask[i].eq(0).view(batch_size, 1), 0) + \ vscore.masked_fill(mask[i].view(batch_size, 1), 0) - + # backtrace batch_idx = torch.arange(batch_size, dtype=torch.long, device=logits.device) seq_idx = torch.arange(seq_len, dtype=torch.long, device=logits.device) lens = (mask.long().sum(0) - 1) # idxes [L, B], batched idx from seq_len-1 to 0 idxes = (lens.view(1, -1) - seq_idx.view(-1, 1)) % seq_len - + ans = logits.new_empty((seq_len, batch_size), dtype=torch.long) ans_score, last_tags = vscore.max(1) ans[idxes[0], batch_idx] = last_tags From a09cf518d7d923c5adae48d3f8f54009682a795b Mon Sep 17 00:00:00 2001 From: ChenXin Date: Fri, 12 Jul 2019 11:05:57 +0800 Subject: [PATCH 006/141] =?UTF-8?q?modules=E5=85=A5=E5=8F=A3=E7=9A=84?= =?UTF-8?q?=E4=BB=8B=E7=BB=8D=E5=92=8Cdropout=E7=9A=84=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/modules/__init__.py | 27 +++++++++++++-------------- fastNLP/modules/dropout.py | 6 ++---- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/fastNLP/modules/__init__.py b/fastNLP/modules/__init__.py index 43ec3f5f..7b0237dc 100644 --- a/fastNLP/modules/__init__.py +++ b/fastNLP/modules/__init__.py @@ -1,22 +1,19 @@ """ -大部分用于的 NLP 任务神经网络都可以看做由编码 :mod:`~fastNLP.modules.encoder` 、 -解码 :mod:`~fastNLP.modules.decoder` 两种模块组成。 .. image:: figures/text_classification.png -:mod:`~fastNLP.modules` 中实现了 fastNLP 提供的诸多模块组件，可以帮助用户快速搭建自己所需的网络。 -两种模块的功能和常见组件如下: +大部分用于的 NLP 任务神经网络都可以看做由 :mod:`embedding` 、 :mod:`~fastNLP.modules.encoder` 、 +:mod:`~fastNLP.modules.decoder` 三种模块组成。 本模块中实现了 fastNLP 提供的诸多模块组件， +可以帮助用户快速搭建自己所需的网络。几种模块的功能和常见组件如下: + +.. csv-table:: + :header: "类型", "功能", "常见组件" + + "embedding", 参见 :doc:`fastNLP.embeddings` , "Elmo, Bert" + "encoder", "将输入编码为具有表示能力的向量", "CNN, LSTM, Transformer" + "decoder", "将具有某种表示意义的向量解码为需要的输出形式 ", "MLP, CRF" + "其它", "配合其它组件使用的组件", "Dropout" -+-----------------------+-----------------------+-----------------------+ -| module type | functionality | example | -+=======================+=======================+=======================+ -| encoder | 将输入编码为具有具 | embedding, RNN, CNN, | -| | 有表示能力的向量 | transformer | -+-----------------------+-----------------------+-----------------------+ -| decoder | 将具有某种表示意义的 | MLP, CRF | -| | 向量解码为需要的输出 | | -| | 形式 | | -+-----------------------+-----------------------+-----------------------+ """ __all__ = [ @@ -40,6 +37,8 @@ __all__ = [ "ConditionalRandomField", "viterbi_decode", "allowed_transitions", + + "TimestepDropout", ] from . import decoder diff --git a/fastNLP/modules/dropout.py b/fastNLP/modules/dropout.py index 1363165c..0ea2a2d9 100644 --- a/fastNLP/modules/dropout.py +++ b/fastNLP/modules/dropout.py @@ -5,10 +5,8 @@ import torch class TimestepDropout(torch.nn.Dropout): """ - 别名：:class:`fastNLP.modules.TimestepDropout` - - 接受的参数shape为``[batch_size, num_timesteps, embedding_dim)]`` 使用同一个mask(shape为``(batch_size, embedding_dim)``) - 在每个timestamp上做dropout。 + 传入参数的shape为 ``(batch_size, num_timesteps, embedding_dim)`` + 使用同一个shape为 ``(batch_size, embedding_dim)`` 的mask在每个timestamp上做dropout。 """ def forward(self, x): From f3a9fc5b7912038bc41cfd6f53515f88405f8b9d Mon Sep 17 00:00:00 2001 From: ChenXin Date: Fri, 12 Jul 2019 11:16:21 +0800 Subject: [PATCH 007/141] =?UTF-8?q?encoder=E9=87=8C=E9=9D=A2=E7=9A=84?= =?UTF-8?q?=E7=BB=93=E6=9E=84=E5=92=8C=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/modules/__init__.py | 14 ++++-- fastNLP/modules/encoder/__init__.py | 12 ++--- fastNLP/modules/encoder/attention.py | 54 ++++++++++----------- fastNLP/modules/encoder/bert.py | 44 ++++++++++------- fastNLP/modules/encoder/char_encoder.py | 23 ++++----- fastNLP/modules/encoder/conv_maxpool.py | 15 +++--- fastNLP/modules/encoder/lstm.py | 5 +- fastNLP/modules/encoder/pooling.py | 30 ++++++------ fastNLP/modules/encoder/star_transformer.py | 50 +++++++++---------- fastNLP/modules/encoder/transformer.py | 10 ++-- fastNLP/modules/encoder/variational_rnn.py | 48 +++++++++--------- 11 files changed, 161 insertions(+), 144 deletions(-) diff --git a/fastNLP/modules/__init__.py b/fastNLP/modules/__init__.py index 7b0237dc..0e16d48c 100644 --- a/fastNLP/modules/__init__.py +++ b/fastNLP/modules/__init__.py @@ -17,22 +17,30 @@ """ __all__ = [ - # "BertModel", + "BertModel", + "ConvolutionCharEncoder", "LSTMCharEncoder", + "ConvMaxpool", + "LSTM", + "StarTransformer", + "TransformerEncoder", + "VarRNN", "VarLSTM", "VarGRU", - + "MaxPool", "MaxPoolWithMask", "AvgPool", + "AvgPoolWithMask", + "MultiHeadAttention", - + "MLP", "ConditionalRandomField", "viterbi_decode", diff --git a/fastNLP/modules/encoder/__init__.py b/fastNLP/modules/encoder/__init__.py index 051a0c01..4946a70f 100644 --- a/fastNLP/modules/encoder/__init__.py +++ b/fastNLP/modules/encoder/__init__.py @@ -1,17 +1,17 @@ __all__ = [ "BertModel", - + "ConvolutionCharEncoder", "LSTMCharEncoder", - + "ConvMaxpool", - + "LSTM", - + "StarTransformer", - + "TransformerEncoder", - + "VarRNN", "VarLSTM", "VarGRU", diff --git a/fastNLP/modules/encoder/attention.py b/fastNLP/modules/encoder/attention.py index 0a42d889..fe3f7fd8 100644 --- a/fastNLP/modules/encoder/attention.py +++ b/fastNLP/modules/encoder/attention.py @@ -8,8 +8,6 @@ import torch import torch.nn.functional as F from torch import nn -from fastNLP.modules.dropout import TimestepDropout - from fastNLP.modules.utils import initial_parameter @@ -18,7 +16,7 @@ class DotAttention(nn.Module): .. todo:: 补上文档 """ - + def __init__(self, key_size, value_size, dropout=0.0): super(DotAttention, self).__init__() self.key_size = key_size @@ -26,7 +24,7 @@ class DotAttention(nn.Module): self.scale = math.sqrt(key_size) self.drop = nn.Dropout(dropout) self.softmax = nn.Softmax(dim=2) - + def forward(self, Q, K, V, mask_out=None): """ @@ -45,7 +43,7 @@ class DotAttention(nn.Module): class MultiHeadAttention(nn.Module): """ - 别名：:class:`fastNLP.modules.MultiHeadAttention` :class:`fastNLP.modules.encoder.attention.MultiHeadAttention` + 别名：:class:`fastNLP.modules.MultiHeadAttention` :class:`fastNLP.modules.encoder.MultiHeadAttention` :param input_size: int, 输入维度的大小。同时也是输出维度的大小。 :param key_size: int, 每个head的维度大小。 @@ -53,14 +51,14 @@ class MultiHeadAttention(nn.Module): :param num_head: int，head的数量。 :param dropout: float。 """ - + def __init__(self, input_size, key_size, value_size, num_head, dropout=0.1): super(MultiHeadAttention, self).__init__() self.input_size = input_size self.key_size = key_size self.value_size = value_size self.num_head = num_head - + in_size = key_size * num_head self.q_in = nn.Linear(input_size, in_size) self.k_in = nn.Linear(input_size, in_size) @@ -69,14 +67,14 @@ class MultiHeadAttention(nn.Module): self.attention = DotAttention(key_size=key_size, value_size=value_size, dropout=dropout) self.out = nn.Linear(value_size * num_head, input_size) self.reset_parameters() - + def reset_parameters(self): sqrt = math.sqrt nn.init.normal_(self.q_in.weight, mean=0, std=sqrt(2.0 / (self.input_size + self.key_size))) nn.init.normal_(self.k_in.weight, mean=0, std=sqrt(2.0 / (self.input_size + self.key_size))) nn.init.normal_(self.v_in.weight, mean=0, std=sqrt(2.0 / (self.input_size + self.value_size))) nn.init.xavier_normal_(self.out.weight) - + def forward(self, Q, K, V, atte_mask_out=None): """ @@ -92,7 +90,7 @@ class MultiHeadAttention(nn.Module): q = self.q_in(Q).view(batch, sq, n_head, d_k) k = self.k_in(K).view(batch, sk, n_head, d_k) v = self.v_in(V).view(batch, sk, n_head, d_v) - + # transpose q, k and v to do batch attention q = q.permute(2, 0, 1, 3).contiguous().view(-1, sq, d_k) k = k.permute(2, 0, 1, 3).contiguous().view(-1, sk, d_k) @@ -100,7 +98,7 @@ class MultiHeadAttention(nn.Module): if atte_mask_out is not None: atte_mask_out = atte_mask_out.repeat(n_head, 1, 1) atte = self.attention(q, k, v, atte_mask_out).view(n_head, batch, sq, d_v) - + # concat all heads, do output linear atte = atte.permute(1, 2, 0, 3).contiguous().view(batch, sq, -1) output = self.out(atte) @@ -124,11 +122,11 @@ class BiAttention(nn.Module): \end{array} """ - + def __init__(self): super(BiAttention, self).__init__() self.inf = 10e12 - + def forward(self, in_x1, in_x2, x1_len, x2_len): """ :param torch.Tensor in_x1: [batch_size, x1_seq_len, hidden_size] 第一句的特征表示 @@ -139,36 +137,36 @@ class BiAttention(nn.Module): torch.Tensor out_x2: [batch_size, x2_seq_len, hidden_size] 第一句attend到的特征表示 """ - + assert in_x1.size()[0] == in_x2.size()[0] assert in_x1.size()[2] == in_x2.size()[2] # The batch size and hidden size must be equal. assert in_x1.size()[1] == x1_len.size()[1] and in_x2.size()[1] == x2_len.size()[1] # The seq len in in_x and x_len must be equal. assert in_x1.size()[0] == x1_len.size()[0] and x1_len.size()[0] == x2_len.size()[0] - + batch_size = in_x1.size()[0] x1_max_len = in_x1.size()[1] x2_max_len = in_x2.size()[1] - + in_x2_t = torch.transpose(in_x2, 1, 2) # [batch_size, hidden_size, x2_seq_len] - + attention_matrix = torch.bmm(in_x1, in_x2_t) # [batch_size, x1_seq_len, x2_seq_len] - + a_mask = x1_len.le(0.5).float() * -self.inf # [batch_size, x1_seq_len] a_mask = a_mask.view(batch_size, x1_max_len, -1) a_mask = a_mask.expand(-1, -1, x2_max_len) # [batch_size, x1_seq_len, x2_seq_len] b_mask = x2_len.le(0.5).float() * -self.inf b_mask = b_mask.view(batch_size, -1, x2_max_len) b_mask = b_mask.expand(-1, x1_max_len, -1) # [batch_size, x1_seq_len, x2_seq_len] - + attention_a = F.softmax(attention_matrix + a_mask, dim=2) # [batch_size, x1_seq_len, x2_seq_len] attention_b = F.softmax(attention_matrix + b_mask, dim=1) # [batch_size, x1_seq_len, x2_seq_len] - + out_x1 = torch.bmm(attention_a, in_x2) # [batch_size, x1_seq_len, hidden_size] attention_b_t = torch.transpose(attention_b, 1, 2) out_x2 = torch.bmm(attention_b_t, in_x1) # [batch_size, x2_seq_len, hidden_size] - + return out_x1, out_x2 @@ -182,10 +180,10 @@ class SelfAttention(nn.Module): :param float drop: dropout概率，默认值为0.5 :param str initial_method: 初始化参数方法 """ - + def __init__(self, input_size, attention_unit=300, attention_hops=10, drop=0.5, initial_method=None, ): super(SelfAttention, self).__init__() - + self.attention_hops = attention_hops self.ws1 = nn.Linear(input_size, attention_unit, bias=False) self.ws2 = nn.Linear(attention_unit, attention_hops, bias=False) @@ -194,7 +192,7 @@ class SelfAttention(nn.Module): self.drop = nn.Dropout(drop) self.tanh = nn.Tanh() initial_parameter(self, initial_method) - + def _penalization(self, attention): """ compute the penalization term for attention module @@ -208,7 +206,7 @@ class SelfAttention(nn.Module): mat = torch.bmm(attention, attention_t) - self.I[:attention.size(0)] ret = (torch.sum(torch.sum((mat ** 2), 2), 1).squeeze() + 1e-10) ** 0.5 return torch.sum(ret) / size[0] - + def forward(self, input, input_origin): """ :param torch.Tensor input: [baz, senLen, h_dim] 要做attention的矩阵 @@ -218,14 +216,14 @@ class SelfAttention(nn.Module): """ input = input.contiguous() size = input.size() # [bsz, len, nhid] - + input_origin = input_origin.expand(self.attention_hops, -1, -1) # [hops,baz, len] input_origin = input_origin.transpose(0, 1).contiguous() # [baz, hops,len] - + y1 = self.tanh(self.ws1(self.drop(input))) # [baz,len,dim] -->[bsz,len, attention-unit] attention = self.ws2(y1).transpose(1, 2).contiguous() # [bsz,len, attention-unit]--> [bsz, len, hop]--> [baz,hop,len] - + attention = attention + (-999999 * (input_origin == 0).float()) # remove the weight on padding token. attention = F.softmax(attention, 2) # [baz ,hop, len] return torch.bmm(attention, input), self._penalization(attention) # output1 --> [baz ,hop ,nhid] diff --git a/fastNLP/modules/encoder/bert.py b/fastNLP/modules/encoder/bert.py index 6d32ae74..ddd22ed0 100644 --- a/fastNLP/modules/encoder/bert.py +++ b/fastNLP/modules/encoder/bert.py @@ -1,11 +1,11 @@ - - - """ 这个页面的代码很大程度上参考(复制粘贴)了https://github.com/huggingface/pytorch-pretrained-BERT的代码， 如果你发现该代码对你 有用，也请引用一下他们。 """ +__all__ = [ + "BertModel" +] import collections @@ -26,6 +26,7 @@ CONFIG_FILE = 'bert_config.json' class BertConfig(object): """Configuration class to store the configuration of a `BertModel`. """ + def __init__(self, vocab_size_or_config_json_file, hidden_size=768, @@ -65,7 +66,7 @@ class BertConfig(object): layer_norm_eps: The epsilon used by LayerNorm. """ if isinstance(vocab_size_or_config_json_file, str) or (sys.version_info[0] == 2 - and isinstance(vocab_size_or_config_json_file, unicode)): + and isinstance(vocab_size_or_config_json_file, unicode)): with open(vocab_size_or_config_json_file, "r", encoding='utf-8') as reader: json_config = json.loads(reader.read()) for key, value in json_config.items(): @@ -150,6 +151,7 @@ class BertLayerNorm(nn.Module): class BertEmbeddings(nn.Module): """Construct the embeddings from word, position and token_type embeddings. """ + def __init__(self, config): super(BertEmbeddings, self).__init__() self.word_embeddings = nn.Embedding(config.vocab_size, config.hidden_size, padding_idx=0) @@ -331,7 +333,10 @@ class BertPooler(nn.Module): class BertModel(nn.Module): - """BERT(Bidirectional Embedding Representations from Transformers). + """ + 别名：:class:`fastNLP.modules.BertModel` :class:`fastNLP.modules.encoder.BertModel` + + BERT(Bidirectional Embedding Representations from Transformers). 如果你想使用预训练好的权重矩阵，请在以下网址下载. sources:: @@ -449,9 +454,9 @@ class BertModel(nn.Module): model = cls(config, *inputs, **kwargs) if state_dict is None: files = glob.glob(os.path.join(pretrained_model_dir, '*.bin')) - if len(files)==0: + if len(files) == 0: raise FileNotFoundError(f"There is no *.bin file in {pretrained_model_dir}") - elif len(files)>1: + elif len(files) > 1: raise FileExistsError(f"There are multiple *.bin files in {pretrained_model_dir}") weights_path = files[0] state_dict = torch.load(weights_path, map_location='cpu') @@ -580,6 +585,7 @@ def load_vocab(vocab_file): index += 1 return vocab + class BasicTokenizer(object): """Runs basic tokenization (punctuation splitting, lower casing, etc.).""" @@ -765,8 +771,8 @@ class BertTokenizer(object): [(ids, tok) for tok, ids in self.vocab.items()]) self.do_basic_tokenize = do_basic_tokenize if do_basic_tokenize: - self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case, - never_split=never_split) + self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case, + never_split=never_split) self.wordpiece_tokenizer = WordpieceTokenizer(vocab=self.vocab) self.max_len = max_len if max_len is not None else int(1e12) @@ -821,7 +827,7 @@ class BertTokenizer(object): for token, token_index in sorted(self.vocab.items(), key=lambda kv: kv[1]): if index != token_index: print("Saving vocabulary to {}: vocabulary indices are not consecutive." - " Please check that the vocabulary is not corrupted!".format(vocab_file)) + " Please check that the vocabulary is not corrupted!".format(vocab_file)) index = token_index writer.write(token + u'\n') index += 1 @@ -841,6 +847,7 @@ class BertTokenizer(object): tokenizer = cls(pretrained_model_name_or_path, *inputs, **kwargs) return tokenizer + VOCAB_NAME = 'vocab.txt' @@ -849,7 +856,8 @@ class _WordPieceBertModel(nn.Module): 这个模块用于直接计算word_piece的结果. """ - def __init__(self, model_dir:str, layers:str='-1'): + + def __init__(self, model_dir: str, layers: str = '-1'): super().__init__() self.tokenzier = BertTokenizer.from_pretrained(model_dir) @@ -858,11 +866,11 @@ class _WordPieceBertModel(nn.Module): encoder_layer_number = len(self.encoder.encoder.layer) self.layers = list(map(int, layers.split(','))) for layer in self.layers: - if layer<0: - assert -layer<=encoder_layer_number, f"The layer index:{layer} is out of scope for " \ + if layer < 0: + assert -layer <= encoder_layer_number, f"The layer index:{layer} is out of scope for " \ f"a bert model with {encoder_layer_number} layers." else: - assert layer