From 7514be6f30cafe6e7e16a1477ad61019985796f0 Mon Sep 17 00:00:00 2001 From: FengZiYjun Date: Wed, 11 Jul 2018 21:51:35 +0800 Subject: [PATCH] - add validation loss into trainer.train - restructure: move reproduction outside - add evaluate in tester --- fastNLP/action/tester.py | 36 +++++++++++++----- fastNLP/action/trainer.py | 2 + fastNLP/reproduction/__init__.py | 0 .../CNN-sentence_classification/.gitignore | 0 .../CNN-sentence_classification/README.md | 0 .../CNN-sentence_classification/__init__.py | 0 .../CNN-sentence_classification/dataset.py | 0 .../CNN-sentence_classification/model.py | 0 .../rt-polaritydata/rt-polarity.neg | 0 .../rt-polaritydata/rt-polarity.pos | 0 .../CNN-sentence_classification/train.py | 0 .../Char-aware_NLM/LICENSE | 0 .../Char-aware_NLM/README.md | 0 .../Char-aware_NLM/__init__.py | 0 .../Char-aware_NLM/model.py | 0 .../Char-aware_NLM/test.py | 0 .../Char-aware_NLM/test.txt | 0 .../Char-aware_NLM/train.py | 0 .../Char-aware_NLM/train.txt | 0 .../Char-aware_NLM/utilities.py | 0 .../Char-aware_NLM/valid.txt | 0 .../HAN-document_classification/README.md | 0 .../HAN-document_classification/__init__.py | 0 .../data/test_samples.pkl | Bin .../data/train_samples.pkl | Bin .../data/yelp.word2vec | Bin .../HAN-document_classification/evaluate.py | 0 .../HAN-document_classification/model.py | 0 .../HAN-document_classification/preprocess.py | 0 .../HAN-document_classification/train.py | 0 30 files changed, 29 insertions(+), 9 deletions(-) delete mode 100644 fastNLP/reproduction/__init__.py rename {fastNLP/reproduction => reproduction}/CNN-sentence_classification/.gitignore (100%) rename {fastNLP/reproduction => reproduction}/CNN-sentence_classification/README.md (100%) rename {fastNLP/reproduction => reproduction}/CNN-sentence_classification/__init__.py (100%) rename {fastNLP/reproduction => reproduction}/CNN-sentence_classification/dataset.py (100%) rename {fastNLP/reproduction => reproduction}/CNN-sentence_classification/model.py (100%) rename {fastNLP/reproduction => reproduction}/CNN-sentence_classification/rt-polaritydata/rt-polarity.neg (100%) rename {fastNLP/reproduction => reproduction}/CNN-sentence_classification/rt-polaritydata/rt-polarity.pos (100%) rename {fastNLP/reproduction => reproduction}/CNN-sentence_classification/train.py (100%) rename {fastNLP/reproduction => reproduction}/Char-aware_NLM/LICENSE (100%) rename {fastNLP/reproduction => reproduction}/Char-aware_NLM/README.md (100%) rename {fastNLP/reproduction => reproduction}/Char-aware_NLM/__init__.py (100%) rename {fastNLP/reproduction => reproduction}/Char-aware_NLM/model.py (100%) rename {fastNLP/reproduction => reproduction}/Char-aware_NLM/test.py (100%) rename {fastNLP/reproduction => reproduction}/Char-aware_NLM/test.txt (100%) rename {fastNLP/reproduction => reproduction}/Char-aware_NLM/train.py (100%) rename {fastNLP/reproduction => reproduction}/Char-aware_NLM/train.txt (100%) rename {fastNLP/reproduction => reproduction}/Char-aware_NLM/utilities.py (100%) rename {fastNLP/reproduction => reproduction}/Char-aware_NLM/valid.txt (100%) rename {fastNLP/reproduction => reproduction}/HAN-document_classification/README.md (100%) rename {fastNLP/reproduction => reproduction}/HAN-document_classification/__init__.py (100%) rename {fastNLP/reproduction => reproduction}/HAN-document_classification/data/test_samples.pkl (100%) rename {fastNLP/reproduction => reproduction}/HAN-document_classification/data/train_samples.pkl (100%) rename {fastNLP/reproduction => reproduction}/HAN-document_classification/data/yelp.word2vec (100%) rename {fastNLP/reproduction => reproduction}/HAN-document_classification/evaluate.py (100%) rename {fastNLP/reproduction => reproduction}/HAN-document_classification/model.py (100%) rename {fastNLP/reproduction => reproduction}/HAN-document_classification/preprocess.py (100%) rename {fastNLP/reproduction => reproduction}/HAN-document_classification/train.py (100%) diff --git a/fastNLP/action/tester.py b/fastNLP/action/tester.py index 2a71cf4d..9d32ec40 100644 --- a/fastNLP/action/tester.py +++ b/fastNLP/action/tester.py @@ -1,5 +1,6 @@ import _pickle +import numpy as np import torch from fastNLP.action.action import Action @@ -16,8 +17,7 @@ class BaseTester(Action): """ super(BaseTester, self).__init__() self.validate_in_training = test_args["validate_in_training"] - self.valid_x = None - self.valid_y = None + self.save_dev_data = None self.save_output = test_args["save_output"] self.output = None self.save_loss = test_args["save_loss"] @@ -26,8 +26,14 @@ class BaseTester(Action): self.pickle_path = test_args["pickle_path"] self.iterator = None + self.model = None + self.eval_history = [] + def test(self, network): # print("--------------testing----------------") + self.model = network + + # turn on the testing mode; clean up the history self.mode(network, test=True) dev_data = self.prepare_input(self.pickle_path) @@ -35,7 +41,6 @@ class BaseTester(Action): self.iterator = iter(Batchifier(RandomSampler(dev_data), self.batch_size, drop_last=True)) batch_output = list() - eval_history = list() num_iter = len(dev_data) // self.batch_size for step in range(num_iter): @@ -47,11 +52,18 @@ class BaseTester(Action): if self.save_output: batch_output.append(prediction) if self.save_loss: - eval_history.append(eval_results) + self.eval_history.append(eval_results) def prepare_input(self, data_path): - data_dev = _pickle.load(open(data_path + "/data_train.pkl", "rb")) - return data_dev + """ + Save the dev data once it is loaded. Can return directly next time. + :param data_path: str, the path to the pickle data for dev + :return save_dev_data: list. Each entry is a sample, which is also a list of features and label(s). + """ + if self.save_dev_data is None: + data_dev = _pickle.load(open(data_path + "/data_train.pkl", "rb")) + self.save_dev_data = data_dev + return self.save_dev_data def batchify(self, data): """ @@ -99,11 +111,12 @@ class BaseTester(Action): raise NotImplementedError def mode(self, model, test=True): - """To do: combine this function with Trainer""" + """To do: combine this function with Trainer ?? """ if test: model.eval() else: model.train() + self.eval_history.clear() class POSTester(BaseTester): @@ -115,6 +128,7 @@ class POSTester(BaseTester): super(POSTester, self).__init__(test_args) self.max_len = None self.mask = None + self.batch_result = None def data_forward(self, network, x): """To Do: combine with Trainer @@ -132,5 +146,9 @@ class POSTester(BaseTester): return y def evaluate(self, predict, truth): - """To Do: """ - return 0 + truth = torch.Tensor(truth) + loss, prediction = self.model.loss(predict, truth, self.mask, self.batch_size, self.max_len) + return loss.data + + def matrices(self): + return np.mean(self.eval_history) diff --git a/fastNLP/action/trainer.py b/fastNLP/action/trainer.py index 034b46ca..0ab9fee7 100644 --- a/fastNLP/action/trainer.py +++ b/fastNLP/action/trainer.py @@ -89,6 +89,7 @@ class BaseTrainer(Action): if data_dev is None: raise RuntimeError("No validation data provided.") validator.test(network) + print("[epoch {}] dev loss={:.2f}".format(epoch, validator.matrices())) # finish training @@ -386,6 +387,7 @@ class POSTrainer(BaseTrainer): else: self.define_loss() loss, prediction = self.loss_func(predict, truth, self.mask, self.batch_size, self.max_len) + # print("loss={:.2f}".format(loss.data)) return loss diff --git a/fastNLP/reproduction/__init__.py b/fastNLP/reproduction/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/fastNLP/reproduction/CNN-sentence_classification/.gitignore b/reproduction/CNN-sentence_classification/.gitignore similarity index 100% rename from fastNLP/reproduction/CNN-sentence_classification/.gitignore rename to reproduction/CNN-sentence_classification/.gitignore diff --git a/fastNLP/reproduction/CNN-sentence_classification/README.md b/reproduction/CNN-sentence_classification/README.md similarity index 100% rename from fastNLP/reproduction/CNN-sentence_classification/README.md rename to reproduction/CNN-sentence_classification/README.md diff --git a/fastNLP/reproduction/CNN-sentence_classification/__init__.py b/reproduction/CNN-sentence_classification/__init__.py similarity index 100% rename from fastNLP/reproduction/CNN-sentence_classification/__init__.py rename to reproduction/CNN-sentence_classification/__init__.py diff --git a/fastNLP/reproduction/CNN-sentence_classification/dataset.py b/reproduction/CNN-sentence_classification/dataset.py similarity index 100% rename from fastNLP/reproduction/CNN-sentence_classification/dataset.py rename to reproduction/CNN-sentence_classification/dataset.py diff --git a/fastNLP/reproduction/CNN-sentence_classification/model.py b/reproduction/CNN-sentence_classification/model.py similarity index 100% rename from fastNLP/reproduction/CNN-sentence_classification/model.py rename to reproduction/CNN-sentence_classification/model.py diff --git a/fastNLP/reproduction/CNN-sentence_classification/rt-polaritydata/rt-polarity.neg b/reproduction/CNN-sentence_classification/rt-polaritydata/rt-polarity.neg similarity index 100% rename from fastNLP/reproduction/CNN-sentence_classification/rt-polaritydata/rt-polarity.neg rename to reproduction/CNN-sentence_classification/rt-polaritydata/rt-polarity.neg diff --git a/fastNLP/reproduction/CNN-sentence_classification/rt-polaritydata/rt-polarity.pos b/reproduction/CNN-sentence_classification/rt-polaritydata/rt-polarity.pos similarity index 100% rename from fastNLP/reproduction/CNN-sentence_classification/rt-polaritydata/rt-polarity.pos rename to reproduction/CNN-sentence_classification/rt-polaritydata/rt-polarity.pos diff --git a/fastNLP/reproduction/CNN-sentence_classification/train.py b/reproduction/CNN-sentence_classification/train.py similarity index 100% rename from fastNLP/reproduction/CNN-sentence_classification/train.py rename to reproduction/CNN-sentence_classification/train.py diff --git a/fastNLP/reproduction/Char-aware_NLM/LICENSE b/reproduction/Char-aware_NLM/LICENSE similarity index 100% rename from fastNLP/reproduction/Char-aware_NLM/LICENSE rename to reproduction/Char-aware_NLM/LICENSE diff --git a/fastNLP/reproduction/Char-aware_NLM/README.md b/reproduction/Char-aware_NLM/README.md similarity index 100% rename from fastNLP/reproduction/Char-aware_NLM/README.md rename to reproduction/Char-aware_NLM/README.md diff --git a/fastNLP/reproduction/Char-aware_NLM/__init__.py b/reproduction/Char-aware_NLM/__init__.py similarity index 100% rename from fastNLP/reproduction/Char-aware_NLM/__init__.py rename to reproduction/Char-aware_NLM/__init__.py diff --git a/fastNLP/reproduction/Char-aware_NLM/model.py b/reproduction/Char-aware_NLM/model.py similarity index 100% rename from fastNLP/reproduction/Char-aware_NLM/model.py rename to reproduction/Char-aware_NLM/model.py diff --git a/fastNLP/reproduction/Char-aware_NLM/test.py b/reproduction/Char-aware_NLM/test.py similarity index 100% rename from fastNLP/reproduction/Char-aware_NLM/test.py rename to reproduction/Char-aware_NLM/test.py diff --git a/fastNLP/reproduction/Char-aware_NLM/test.txt b/reproduction/Char-aware_NLM/test.txt similarity index 100% rename from fastNLP/reproduction/Char-aware_NLM/test.txt rename to reproduction/Char-aware_NLM/test.txt diff --git a/fastNLP/reproduction/Char-aware_NLM/train.py b/reproduction/Char-aware_NLM/train.py similarity index 100% rename from fastNLP/reproduction/Char-aware_NLM/train.py rename to reproduction/Char-aware_NLM/train.py diff --git a/fastNLP/reproduction/Char-aware_NLM/train.txt b/reproduction/Char-aware_NLM/train.txt similarity index 100% rename from fastNLP/reproduction/Char-aware_NLM/train.txt rename to reproduction/Char-aware_NLM/train.txt diff --git a/fastNLP/reproduction/Char-aware_NLM/utilities.py b/reproduction/Char-aware_NLM/utilities.py similarity index 100% rename from fastNLP/reproduction/Char-aware_NLM/utilities.py rename to reproduction/Char-aware_NLM/utilities.py diff --git a/fastNLP/reproduction/Char-aware_NLM/valid.txt b/reproduction/Char-aware_NLM/valid.txt similarity index 100% rename from fastNLP/reproduction/Char-aware_NLM/valid.txt rename to reproduction/Char-aware_NLM/valid.txt diff --git a/fastNLP/reproduction/HAN-document_classification/README.md b/reproduction/HAN-document_classification/README.md similarity index 100% rename from fastNLP/reproduction/HAN-document_classification/README.md rename to reproduction/HAN-document_classification/README.md diff --git a/fastNLP/reproduction/HAN-document_classification/__init__.py b/reproduction/HAN-document_classification/__init__.py similarity index 100% rename from fastNLP/reproduction/HAN-document_classification/__init__.py rename to reproduction/HAN-document_classification/__init__.py diff --git a/fastNLP/reproduction/HAN-document_classification/data/test_samples.pkl b/reproduction/HAN-document_classification/data/test_samples.pkl similarity index 100% rename from fastNLP/reproduction/HAN-document_classification/data/test_samples.pkl rename to reproduction/HAN-document_classification/data/test_samples.pkl diff --git a/fastNLP/reproduction/HAN-document_classification/data/train_samples.pkl b/reproduction/HAN-document_classification/data/train_samples.pkl similarity index 100% rename from fastNLP/reproduction/HAN-document_classification/data/train_samples.pkl rename to reproduction/HAN-document_classification/data/train_samples.pkl diff --git a/fastNLP/reproduction/HAN-document_classification/data/yelp.word2vec b/reproduction/HAN-document_classification/data/yelp.word2vec similarity index 100% rename from fastNLP/reproduction/HAN-document_classification/data/yelp.word2vec rename to reproduction/HAN-document_classification/data/yelp.word2vec diff --git a/fastNLP/reproduction/HAN-document_classification/evaluate.py b/reproduction/HAN-document_classification/evaluate.py similarity index 100% rename from fastNLP/reproduction/HAN-document_classification/evaluate.py rename to reproduction/HAN-document_classification/evaluate.py diff --git a/fastNLP/reproduction/HAN-document_classification/model.py b/reproduction/HAN-document_classification/model.py similarity index 100% rename from fastNLP/reproduction/HAN-document_classification/model.py rename to reproduction/HAN-document_classification/model.py diff --git a/fastNLP/reproduction/HAN-document_classification/preprocess.py b/reproduction/HAN-document_classification/preprocess.py similarity index 100% rename from fastNLP/reproduction/HAN-document_classification/preprocess.py rename to reproduction/HAN-document_classification/preprocess.py diff --git a/fastNLP/reproduction/HAN-document_classification/train.py b/reproduction/HAN-document_classification/train.py similarity index 100% rename from fastNLP/reproduction/HAN-document_classification/train.py rename to reproduction/HAN-document_classification/train.py