Browse Source

Update vocabulary.py (#325)

strip只需要将\n去掉,否则会将一些特殊字符去掉,造成split的时候长度出错
tags/v0.6.0
Gosicfly GitHub 4 years ago
parent
commit
0c990e7b50
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 2 additions and 2 deletions
  1. +2
    -2
      fastNLP/core/vocabulary.py

+ 2
- 2
fastNLP/core/vocabulary.py View File

@@ -540,7 +540,7 @@ class Vocabulary(object):

vocab = Vocabulary()
for line in f:
line = line.strip()
line = line.strip('\n')
if line:
name, value = line.split()
if name in ('max_size', 'min_freq'):
@@ -557,7 +557,7 @@ class Vocabulary(object):
no_create_entry_counter = {}
word2idx = {}
for line in f:
line = line.strip()
line = line.strip('\n')
if line:
parts = line.split('\t')
word,count,idx,no_create_entry = parts[0], int(parts[1]), int(parts[2]), int(parts[3])


Loading…
Cancel
Save