|
|
@@ -97,7 +97,8 @@ class BasePreprocess(object): |
|
|
|
save_pickle(data_train, pickle_path, "data_train.pkl") |
|
|
|
else: |
|
|
|
data_train = load_pickle(pickle_path, "data_train.pkl") |
|
|
|
data_dev = load_pickle(pickle_path, "data_dev.pkl") |
|
|
|
if pickle_exist(pickle_path, "data_dev.pkl"): |
|
|
|
data_dev = load_pickle(pickle_path, "data_dev.pkl") |
|
|
|
else: |
|
|
|
# cross_val is True |
|
|
|
if not pickle_exist(pickle_path, "data_train_0.pkl"): |
|
|
@@ -307,4 +308,4 @@ def infer_preprocess(pickle_path, data): |
|
|
|
data_index = [] |
|
|
|
for example in data: |
|
|
|
data_index.append([word2index.get(w, DEFAULT_UNKNOWN_LABEL) for w in example]) |
|
|
|
return data_index |
|
|
|
return data_index |