Browse Source

check if data_dev.pkl exists

In line 100, add a if statement to check whether there exists a file named "data_dev.pkl" in the pickle path. If not, the file won't be loaded.
tags/v0.1.0
Coet GitHub 6 years ago
parent
commit
50b5345539
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 3 additions and 2 deletions
  1. +3
    -2
      fastNLP/core/preprocess.py

+ 3
- 2
fastNLP/core/preprocess.py View File

@@ -97,7 +97,8 @@ class BasePreprocess(object):
save_pickle(data_train, pickle_path, "data_train.pkl") save_pickle(data_train, pickle_path, "data_train.pkl")
else: else:
data_train = load_pickle(pickle_path, "data_train.pkl") data_train = load_pickle(pickle_path, "data_train.pkl")
data_dev = load_pickle(pickle_path, "data_dev.pkl")
if pickle_exist(pickle_path, "data_dev.pkl"):
data_dev = load_pickle(pickle_path, "data_dev.pkl")
else: else:
# cross_val is True # cross_val is True
if not pickle_exist(pickle_path, "data_train_0.pkl"): if not pickle_exist(pickle_path, "data_train_0.pkl"):
@@ -307,4 +308,4 @@ def infer_preprocess(pickle_path, data):
data_index = [] data_index = []
for example in data: for example in data:
data_index.append([word2index.get(w, DEFAULT_UNKNOWN_LABEL) for w in example]) data_index.append([word2index.get(w, DEFAULT_UNKNOWN_LABEL) for w in example])
return data_index
return data_index

Loading…
Cancel
Save