Browse Source

修复Vocabulary在建好词表之后新加入词导致的pad index错乱的问题

tags/v0.4.10
yh 6 years ago
parent
commit
a40f57ae4a
1 changed files with 4 additions and 4 deletions
  1. +4
    -4
      fastNLP/core/vocabulary.py

+ 4
- 4
fastNLP/core/vocabulary.py View File

@@ -136,10 +136,10 @@ class Vocabulary(object):
""" """
if self.word2idx is None: if self.word2idx is None:
self.word2idx = {} self.word2idx = {}
if self.padding is not None:
self.word2idx[self.padding] = len(self.word2idx)
if self.unknown is not None:
self.word2idx[self.unknown] = len(self.word2idx)
if self.padding is not None:
self.word2idx[self.padding] = len(self.word2idx)
if self.unknown is not None:
self.word2idx[self.unknown] = len(self.word2idx)
max_size = min(self.max_size, len(self.word_count)) if self.max_size else None max_size = min(self.max_size, len(self.word_count)) if self.max_size else None
words = self.word_count.most_common(max_size) words = self.word_count.most_common(max_size)


Loading…
Cancel
Save