From 89ad1954113b485638c8a784d8bd2bf0e0603e78 Mon Sep 17 00:00:00 2001
From: yh_cc <poemsmileyh@gmail.com>
Date: Mon, 26 Oct 2020 14:34:33 +0800
Subject: [PATCH] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=5F=5Fall=5F=5F=E5=B1=9E?=
 =?UTF-8?q?=E6=80=A7?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 fastNLP/core/collate_fn.py                   | 2 ++
 fastNLP/embeddings/transformers_embedding.py | 6 +++++-
 fastNLP/models/seq2seq_generator.py          | 3 +++
 fastNLP/models/seq2seq_model.py              | 3 +++
 fastNLP/modules/decoder/seq2seq_decoder.py   | 3 +++
 fastNLP/modules/encoder/seq2seq_encoder.py   | 3 +++
 6 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/fastNLP/core/collate_fn.py b/fastNLP/core/collate_fn.py
index 7d7f9726..403af270 100644
--- a/fastNLP/core/collate_fn.py
+++ b/fastNLP/core/collate_fn.py
@@ -7,6 +7,8 @@ from .field import _get_ele_type_and_dim
 from .utils import logger
 from copy import deepcopy
 
+__all__ = ['ConcatCollateFn']
+
 
 def _check_type(batch_dict, fields):
     if len(fields) == 0:
diff --git a/fastNLP/embeddings/transformers_embedding.py b/fastNLP/embeddings/transformers_embedding.py
index 4d84fefd..46282af7 100644
--- a/fastNLP/embeddings/transformers_embedding.py
+++ b/fastNLP/embeddings/transformers_embedding.py
@@ -15,6 +15,9 @@ from ..core import logger
 from ..core.vocabulary import Vocabulary
 
 
+__all__ = ['TransformersEmbedding', 'TransformersWordPieceEncoder']
+
+
 class TransformersEmbedding(ContextualEmbedding):
     r"""
     使用transformers中的模型对words进行编码的Embedding。建议将输入的words长度限制在430以内，而不要使用512(根据预训练模型参数，可能有变化)。这是由于
@@ -280,7 +283,7 @@ class _TransformersWordModel(nn.Module):
                 word = tokenizer.unk_token
             elif vocab.word_count[word]<min_freq:
                 word = tokenizer.unk_token
-            word_pieces = self.tokenizer.tokenize(word)
+            word_pieces = self.tokenizer.tokenize(word, add_prefix_space=True)
             word_pieces = self.tokenizer.convert_tokens_to_ids(word_pieces)
             word_to_wordpieces.append(word_pieces)
             word_pieces_lengths.append(len(word_pieces))
@@ -453,6 +456,7 @@ class _WordPieceTransformersModel(nn.Module):
         :return:
         """
         kwargs['add_special_tokens'] = kwargs.get('add_special_tokens', True)
+        kwargs['add_prefix_space'] = kwargs.get('add_special_tokens', True)
 
         encode_func = partial(self.tokenizer.encode, **kwargs)
 
diff --git a/fastNLP/models/seq2seq_generator.py b/fastNLP/models/seq2seq_generator.py
index 33da67bd..cc394c13 100644
--- a/fastNLP/models/seq2seq_generator.py
+++ b/fastNLP/models/seq2seq_generator.py
@@ -6,6 +6,9 @@ from .seq2seq_model import Seq2SeqModel
 from ..modules.generator.seq2seq_generator import SequenceGenerator
 
 
+__all__ = ['SequenceGeneratorModel']
+
+
 class SequenceGeneratorModel(nn.Module):
     """
     用于封装Seq2SeqModel使其可以做生成任务
diff --git a/fastNLP/models/seq2seq_model.py b/fastNLP/models/seq2seq_model.py
index ce867c0b..3f16d74b 100644
--- a/fastNLP/models/seq2seq_model.py
+++ b/fastNLP/models/seq2seq_model.py
@@ -12,6 +12,9 @@ from ..modules.decoder.seq2seq_decoder import Seq2SeqDecoder, TransformerSeq2Seq
 from ..modules.encoder.seq2seq_encoder import Seq2SeqEncoder, TransformerSeq2SeqEncoder, LSTMSeq2SeqEncoder
 
 
+__all__ = ['Seq2SeqModel', 'TransformerSeq2SeqModel', 'LSTMSeq2SeqModel']
+
+
 class Seq2SeqModel(nn.Module):
     def __init__(self, encoder: Seq2SeqEncoder, decoder: Seq2SeqDecoder):
         """
diff --git a/fastNLP/modules/decoder/seq2seq_decoder.py b/fastNLP/modules/decoder/seq2seq_decoder.py
index 2c223dea..482938c3 100644
--- a/fastNLP/modules/decoder/seq2seq_decoder.py
+++ b/fastNLP/modules/decoder/seq2seq_decoder.py
@@ -11,6 +11,9 @@ from ...embeddings.utils import get_embeddings
 from .seq2seq_state import State, LSTMState, TransformerState
 
 
+__all__ = ['Seq2SeqDecoder', 'TransformerSeq2SeqDecoder', 'LSTMSeq2SeqDecoder']
+
+
 class Seq2SeqDecoder(nn.Module):
     """
     Sequence-to-Sequence Decoder的基类。一定需要实现forward函数，剩下的函数根据需要实现。每个Seq2SeqDecoder都应该有相应的State对象
diff --git a/fastNLP/modules/encoder/seq2seq_encoder.py b/fastNLP/modules/encoder/seq2seq_encoder.py
index b35ca2d5..d280582a 100644
--- a/fastNLP/modules/encoder/seq2seq_encoder.py
+++ b/fastNLP/modules/encoder/seq2seq_encoder.py
@@ -12,6 +12,9 @@ from ...embeddings import StaticEmbedding
 from ...embeddings.utils import get_embeddings
 
 
+__all__ = ['Seq2SeqEncoder', 'TransformerSeq2SeqEncoder', 'LSTMSeq2SeqEncoder']
+
+
 class Seq2SeqEncoder(nn.Module):
     """
     所有Sequence2Sequence Encoder的基类。需要实现forward函数