From 4886dbff8b38e9f5167f3691a4ec68db94cb1a06 Mon Sep 17 00:00:00 2001 From: yh_cc Date: Tue, 5 Jan 2021 19:31:00 +0800 Subject: [PATCH] =?UTF-8?q?1.trainer=E5=A2=9E=E5=8A=A0=E5=BC=82=E5=B8=B8?= =?UTF-8?q?=E6=97=B6=E8=BE=93=E5=87=BA=E5=BD=93=E5=89=8Dbatch=E7=9A=84indi?= =?UTF-8?q?ce;=202.BERTEmbedding=E4=BF=AE=E6=94=B9=E4=B8=80=E4=B8=AAtypo;?= =?UTF-8?q?=203.Loss=E5=87=BD=E6=95=B0=E8=BF=9B=E8=A1=8Cupdate;4.seq2seq?= =?UTF-8?q?=5Fgenerator=E4=BF=AE=E6=94=B9bias=E9=98=B2=E6=AD=A2=E6=B2=A1?= =?UTF-8?q?=E6=9C=89=E8=BE=93=E5=87=BAeos?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/core/losses.py | 52 +++++++++++++++---- fastNLP/core/sampler.py | 8 ++- fastNLP/core/trainer.py | 8 ++- fastNLP/embeddings/bert_embedding.py | 28 +++++----- .../modules/generator/seq2seq_generator.py | 2 +- 5 files changed, 72 insertions(+), 26 deletions(-) diff --git a/fastNLP/core/losses.py b/fastNLP/core/losses.py index 642c8ef3..7c671be7 100644 --- a/fastNLP/core/losses.py +++ b/fastNLP/core/losses.py @@ -217,7 +217,7 @@ class CrossEntropyLoss(LossBase): 或(batch_size, num_classes, max_len)， CrossEntropyLoss需要知道哪一维是class的维度以计算loss。如果为-1，就根据pred的第 二维是否等于target的第二维来判断是否需要交换pred的第二维和第三维，因为target的第二维是length的维度，如果这一维度上和pred相等， 那么pred可能第二维也是长度维(存在误判的可能，如果有误判的情况，请显示设置该值)。其它大于0的值则认为该维度是class的维度。 - :param padding_idx: padding的index，在计算loss时将忽略target中标号为padding_idx的内容, 可以通过该值代替 + :param ignore_idx: padding的index，在计算loss时将忽略target中标号为padding_idx的内容, 可以通过该值代替 传入seq_len. :param str reduction: 支持 `mean` ，`sum` 和 `none` . @@ -227,10 +227,11 @@ class CrossEntropyLoss(LossBase): """ - def __init__(self, pred=None, target=None, seq_len=None, class_in_dim=-1, padding_idx=-100, reduction='mean'): + def __init__(self, pred=None, target=None, seq_len=None, class_in_dim=-1, ignore_idx=-100, reduction='mean', **kwargs): super(CrossEntropyLoss, self).__init__() self._init_param_map(pred=pred, target=target, seq_len=seq_len) - self.padding_idx = padding_idx + ignore_idx = kwargs.pop('padding_idx', ignore_idx) + self.ignore_idx = ignore_idx assert reduction in ('mean', 'sum', 'none') self.reduction = reduction self.class_in_dim = class_in_dim @@ -238,7 +239,7 @@ class CrossEntropyLoss(LossBase): def get_loss(self, pred, target, seq_len=None): if seq_len is not None and target.dim()>1: mask = seq_len_to_mask(seq_len, max_len=target.size(1)).eq(False) - target = target.masked_fill(mask, self.padding_idx) + target = target.masked_fill(mask, self.ignore_idx) if pred.dim() > 2: if self.class_in_dim == -1: @@ -250,7 +251,7 @@ class CrossEntropyLoss(LossBase): target = target.reshape(-1) return F.cross_entropy(input=pred, target=target, - ignore_index=self.padding_idx, reduction=self.reduction) + ignore_index=self.ignore_idx, reduction=self.reduction) class L1Loss(LossBase): @@ -318,16 +319,30 @@ class BCEWithLogits(LossBase): :param pred: 参数映射表中 `pred` 的映射关系，None表示映射关系为 `pred` -> `pred` :param target: 参数映射表中 `target` 的映射关系，None表示映射关系为 `target` -> `target` + :param int class_in_dim: 在序列标注的场景中，pred可能的shape为(batch_size, max_len, num_classes) + 或(batch_size, num_classes, max_len)， CrossEntropyLoss需要知道哪一维是class的维度以计算loss。如果为-1，就根据pred的第 + 二维是否等于target的第二维来判断是否需要交换pred的第二维和第三维，因为target的第二维是length的维度，如果这一维度上和pred相等， + 那么pred可能第二维也是长度维(存在误判的可能，如果有误判的情况，请显示设置该值)。其它大于0的值则认为该维度是class的维度。 :param str reduction: 支持 `mean` ，`sum` 和 `none` . """ - def __init__(self, pred=None, target=None, reduction='mean'): + def __init__(self, pred=None, target=None, class_in_dim=-1, reduction='mean'): super(BCEWithLogits, self).__init__() self._init_param_map(pred=pred, target=target) assert reduction in ('mean', 'sum', 'none') self.reduction = reduction + self.class_in_dim = class_in_dim def get_loss(self, pred, target): + if pred.dim() > 2: + if self.class_in_dim == -1: + if pred.size(1) != target.size(1): # 有可能顺序替换了 + pred = pred.transpose(1, 2) + else: + pred = pred.transpose(-1, self.class_in_dim) + pred = pred.reshape(-1, pred.size(-1)) + target = target.reshape(-1) + return F.binary_cross_entropy_with_logits(input=pred, target=target, reduction=self.reduction) @@ -336,22 +351,41 @@ class NLLLoss(LossBase): 负对数似然损失函数 """ - def __init__(self, pred=None, target=None, ignore_idx=-100, reduction='mean'): + def __init__(self, pred=None, target=None, seq_len=None, class_in_dim=-1, ignore_idx=-100, reduction='mean'): r""" :param pred: 参数映射表中 `pred` 的映射关系，None表示映射关系为 `pred` -> `pred` :param target: 参数映射表中 `target` 的映射关系，None表示映射关系为 `target` -> `target` + :param seq_len: 句子的长度, 长度之外的token不会计算loss。仅在输出为3d时需要 + :param int class_in_dim: 在序列标注的场景中，pred可能的shape为(batch_size, max_len, num_classes) + 或(batch_size, num_classes, max_len)， CrossEntropyLoss需要知道哪一维是class的维度以计算loss。如果为-1，就根据pred的第 + 二维是否等于target的第二维来判断是否需要交换pred的第二维和第三维，因为target的第二维是length的维度，如果这一维度上和pred相等， + 那么pred可能第二维也是长度维(存在误判的可能，如果有误判的情况，请显示设置该值)。其它大于0的值则认为该维度是class的维度。 :param ignore_idx: ignore的index，在计算loss时将忽略target中标号为ignore_idx的内容, 可以通过该值代替 传入seq_len. :param str reduction: 支持 `mean` ，`sum` 和 `none` . """ super(NLLLoss, self).__init__() - self._init_param_map(pred=pred, target=target) + self._init_param_map(pred=pred, target=target, seq_len=seq_len) assert reduction in ('mean', 'sum', 'none') self.reduction = reduction self.ignore_idx = ignore_idx + self.class_in_dim = class_in_dim - def get_loss(self, pred, target): + def get_loss(self, pred, target, seq_len=None): + if seq_len is not None and target.dim()>1: + mask = seq_len_to_mask(seq_len, max_len=target.size(1)).eq(False) + target = target.masked_fill(mask, self.ignore_idx) + + if pred.dim() > 2: + if self.class_in_dim == -1: + if pred.size(1) != target.size(1): # 有可能顺序替换了 + pred = pred.transpose(1, 2) + else: + pred = pred.transpose(-1, self.class_in_dim) + pred = pred.reshape(-1, pred.size(-1)) + target = target.reshape(-1) + return F.nll_loss(input=pred, target=target, ignore_index=self.ignore_idx, reduction=self.reduction) diff --git a/fastNLP/core/sampler.py b/fastNLP/core/sampler.py index 329de742..8ad10e26 100644 --- a/fastNLP/core/sampler.py +++ b/fastNLP/core/sampler.py @@ -322,7 +322,8 @@ class SortedSampler(Sampler): def __init__(self, seq_len_field_name='seq_len', descending=True): """ - :param str seq_len_field_name: 对应序列长度的 `field` 的名字 + :param str seq_len_field_name: 按哪个field进行排序。如果传入的field是数字，则直接按照该数字大小排序；如果传入的field不是 + 数字，则使用该field的长度进行排序 :param bool descending: 是否降序排列 """ self.seq_len_field_name = seq_len_field_name @@ -330,6 +331,11 @@ class SortedSampler(Sampler): def __call__(self, data_set): seq_lens = data_set.get_field(self.seq_len_field_name).content + try: + seq_lens = list(map(len, seq_lens)) + except: + pass + orders = np.argsort(seq_lens).tolist() # 从小到大的顺序 if self.descending: orders = orders[::-1] diff --git a/fastNLP/core/trainer.py b/fastNLP/core/trainer.py index a2b9e8dd..628b9711 100644 --- a/fastNLP/core/trainer.py +++ b/fastNLP/core/trainer.py @@ -523,6 +523,7 @@ class Trainer(object): self._forward_func = self.model.forward self.fp16 = fp16 + self.verbose = kwargs.get('verbose', 0) # check fp16相关的设置 self.auto_cast, _grad_scaler = _build_fp16_env(dummy=not fp16) @@ -608,7 +609,7 @@ class Trainer(object): self.callback_manager = CallbackManager(env={"trainer": self}, callbacks=callbacks) - def train(self, load_best_model=True, on_exception='auto'): + def train(self, load_best_model=True, on_exception='auto', **kwargs): r""" 使用该函数使Trainer开始训练。 @@ -617,6 +618,8 @@ class Trainer(object): :param str on_exception: 在训练过程遭遇exception，并被 :py:class:Callback 的on_exception()处理后，是否继续抛出异常。 支持'ignore','raise', 'auto': 'ignore'将捕获异常，写在Trainer.train()后面的代码将继续运行; 'raise'将异常抛出; 'auto'将ignore以下两种Exception: CallbackException与KeyboardInterrupt, raise其它exception. + :param kwargs: + int verbose: 为1时在发生异常时会打印异常发生时batch中的数据在dataset中的index :return dict: 返回一个字典类型的数据, 内含以下内容:: @@ -629,6 +632,7 @@ class Trainer(object): """ results = {} + verbose = kwargs.get('verbose', 0) if self.n_epochs <= 0: self.logger.info(f"training epoch is {self.n_epochs}, nothing was done.") results['seconds'] = 0. @@ -650,6 +654,8 @@ class Trainer(object): except BaseException as e: self.callback_manager.on_exception(e) + if verbose>0: + self.logger.info(f"The data indices for current batch are: {self.data_iterator.cur_batch_indices}.") if on_exception == 'auto': if not isinstance(e, (CallbackException, KeyboardInterrupt)): raise e diff --git a/fastNLP/embeddings/bert_embedding.py b/fastNLP/embeddings/bert_embedding.py index 6434cc0d..01e646a7 100644 --- a/fastNLP/embeddings/bert_embedding.py +++ b/fastNLP/embeddings/bert_embedding.py @@ -393,7 +393,7 @@ class _BertWordModel(nn.Module): else: pos_num_output_layer = max(layer, pos_num_output_layer) - self.tokenzier = BertTokenizer.from_pretrained(model_dir_or_name) + self.tokenizer = BertTokenizer.from_pretrained(model_dir_or_name) self.encoder = BertModel.from_pretrained(model_dir_or_name, neg_num_output_layer=neg_num_output_layer, pos_num_output_layer=pos_num_output_layer, @@ -432,14 +432,14 @@ class _BertWordModel(nn.Module): word = '[UNK]' elif vocab.word_count[word] < min_freq: word = '[UNK]' - word_pieces = self.tokenzier.wordpiece_tokenizer.tokenize(word) - word_pieces = self.tokenzier.convert_tokens_to_ids(word_pieces) + word_pieces = self.tokenizer.wordpiece_tokenizer.tokenize(word) + word_pieces = self.tokenizer.convert_tokens_to_ids(word_pieces) word_to_wordpieces.append(word_pieces) word_pieces_lengths.append(len(word_pieces)) - self._cls_index = self.tokenzier.vocab['[CLS]'] - self._sep_index = self.tokenzier.vocab['[SEP]'] + self._cls_index = self.tokenizer.vocab['[CLS]'] + self._sep_index = self.tokenizer.vocab['[SEP]'] self._word_pad_index = vocab.padding_idx - self._wordpiece_pad_index = self.tokenzier.vocab['[PAD]'] # 需要用于生成word_piece + self._wordpiece_pad_index = self.tokenizer.vocab['[PAD]'] # 需要用于生成word_piece self.word_to_wordpieces = np.array(word_to_wordpieces, dtype=object) self.register_buffer('word_pieces_lengths', torch.LongTensor(word_pieces_lengths)) logger.debug("Successfully generate word pieces.") @@ -566,7 +566,7 @@ class _BertWordModel(nn.Module): :param str folder: :return: """ - self.tokenzier.save_pretrained(folder) + self.tokenizer.save_pretrained(folder) self.encoder.save_pretrained(folder) @@ -579,7 +579,7 @@ class _BertWordPieceModel(nn.Module): def __init__(self, model_dir_or_name: str, layers: str = '-1', pooled_cls: bool=False): super().__init__() - self.tokenzier = BertTokenizer.from_pretrained(model_dir_or_name) + self.tokenizer = BertTokenizer.from_pretrained(model_dir_or_name) self.encoder = BertModel.from_pretrained(model_dir_or_name) # 检查encoder_layer_number是否合理 encoder_layer_number = len(self.encoder.encoder.layer) @@ -599,10 +599,10 @@ class _BertWordPieceModel(nn.Module): assert layer <= encoder_layer_number, f"The layer index:{layer} is out of scope for " \ f"a bert model with {encoder_layer_number} layers." - self._cls_index = self.tokenzier.cls_index - self._sep_index = self.tokenzier.sep_index - self._wordpiece_unknown_index = self.tokenzier.unk_index - self._wordpiece_pad_index = self.tokenzier.pad_index # 需要用于生成word_piece + self._cls_index = self.tokenizer.cls_index + self._sep_index = self.tokenizer.sep_index + self._wordpiece_unknown_index = self.tokenizer.unk_index + self._wordpiece_pad_index = self.tokenizer.pad_index # 需要用于生成word_piece self.pooled_cls = pooled_cls def index_datasets(self, *datasets, field_name, add_cls_sep=True): @@ -615,7 +615,7 @@ class _BertWordPieceModel(nn.Module): :return: """ - encode_func = partial(self.tokenzier.encode, add_special_tokens=add_cls_sep) + encode_func = partial(self.tokenizer.encode, add_special_tokens=add_cls_sep) for index, dataset in enumerate(datasets): try: @@ -654,5 +654,5 @@ class _BertWordPieceModel(nn.Module): :param folder: :return: """ - self.tokenzier.save_pretrained(folder) + self.tokenizer.save_pretrained(folder) self.encoder.save_pretrained(folder) diff --git a/fastNLP/modules/generator/seq2seq_generator.py b/fastNLP/modules/generator/seq2seq_generator.py index 0ba9c02a..c4e0cd87 100644 --- a/fastNLP/modules/generator/seq2seq_generator.py +++ b/fastNLP/modules/generator/seq2seq_generator.py @@ -328,7 +328,7 @@ def _beam_search_generate(decoder: Seq2SeqDecoder, tokens=None, state=None, max_ max_len_eos_mask = max_lengths.eq(cur_len+1) eos_scores = scores[:, _eos_token_id] # 如果已经达到最大长度，就把eos的分数加大 - scores[:, _eos_token_id] = torch.where(max_len_eos_mask, eos_scores+1e12, eos_scores) + scores[:, _eos_token_id] = torch.where(max_len_eos_mask, eos_scores+1e32, eos_scores) if do_sample: if temperature > 0 and temperature != 1: