diff --git a/fastNLP/core/tester.py b/fastNLP/core/tester.py index 24aac951..51f84691 100644 --- a/fastNLP/core/tester.py +++ b/fastNLP/core/tester.py @@ -17,9 +17,9 @@ class Tester(object): """ super(Tester, self).__init__() """ - "default_args" provides default value for important settings. - The initialization arguments "kwargs" with the same key (name) will override the default value. - "kwargs" must have the same type as "default_args" on corresponding keys. + "default_args" provides default value for important settings. + The initialization arguments "kwargs" with the same key (name) will override the default value. + "kwargs" must have the same type as "default_args" on corresponding keys. Otherwise, error will raise. """ default_args = {"batch_size": 8, @@ -29,8 +29,8 @@ class Tester(object): "evaluator": Evaluator() } """ - "required_args" is the collection of arguments that users must pass to Trainer explicitly. - This is used to warn users of essential settings in the training. + "required_args" is the collection of arguments that users must pass to Trainer explicitly. + This is used to warn users of essential settings in the training. Specially, "required_args" does not have default value, so they have nothing to do with "default_args". """ required_args = {} @@ -76,14 +76,17 @@ class Tester(object): data_iterator = Batch(dev_data, self.batch_size, sampler=RandomSampler(), use_cuda=self.use_cuda) - for batch_x, batch_y in data_iterator: - with torch.no_grad(): + with torch.no_grad(): + for batch_x, batch_y in data_iterator: prediction = self.data_forward(network, batch_x) - output_list.append(prediction) - truth_list.append(batch_y) - eval_results = self.evaluate(output_list, truth_list) + output_list.append(prediction) + truth_list.append(batch_y) + eval_results = self.evaluate(output_list, truth_list) print("[tester] {}".format(self.print_eval_results(eval_results))) logger.info("[tester] {}".format(self.print_eval_results(eval_results))) + self.mode(network, is_test=False) + self.metrics = eval_results + return eval_results def mode(self, model, is_test=False): """Train mode or Test mode. This is for PyTorch currently. diff --git a/fastNLP/core/trainer.py b/fastNLP/core/trainer.py index a180b10d..49761725 100644 --- a/fastNLP/core/trainer.py +++ b/fastNLP/core/trainer.py @@ -35,20 +35,21 @@ class Trainer(object): super(Trainer, self).__init__() """ - "default_args" provides default value for important settings. - The initialization arguments "kwargs" with the same key (name) will override the default value. - "kwargs" must have the same type as "default_args" on corresponding keys. + "default_args" provides default value for important settings. + The initialization arguments "kwargs" with the same key (name) will override the default value. + "kwargs" must have the same type as "default_args" on corresponding keys. Otherwise, error will raise. """ default_args = {"epochs": 1, "batch_size": 2, "validate": False, "use_cuda": False, "pickle_path": "./save/", "save_best_dev": False, "model_name": "default_model_name.pkl", "print_every_step": 1, + "valid_step": 500, "eval_sort_key": None, "loss": Loss(None), # used to pass type check "optimizer": Optimizer("Adam", lr=0.001, weight_decay=0), "evaluator": Evaluator() } """ - "required_args" is the collection of arguments that users must pass to Trainer explicitly. - This is used to warn users of essential settings in the training. + "required_args" is the collection of arguments that users must pass to Trainer explicitly. + This is used to warn users of essential settings in the training. Specially, "required_args" does not have default value, so they have nothing to do with "default_args". """ required_args = {} @@ -70,16 +71,20 @@ class Trainer(object): else: # Trainer doesn't care about extra arguments pass - print(default_args) + print("Training Args {}".format(default_args)) + logger.info("Training Args {}".format(default_args)) - self.n_epochs = default_args["epochs"] - self.batch_size = default_args["batch_size"] + self.n_epochs = int(default_args["epochs"]) + self.batch_size = int(default_args["batch_size"]) self.pickle_path = default_args["pickle_path"] self.validate = default_args["validate"] self.save_best_dev = default_args["save_best_dev"] self.use_cuda = default_args["use_cuda"] self.model_name = default_args["model_name"] - self.print_every_step = default_args["print_every_step"] + self.print_every_step = int(default_args["print_every_step"]) + self.valid_step = int(default_args["valid_step"]) + if self.validate is not None: + assert self.valid_step > 0 self._model = None self._loss_func = default_args["loss"].get() # return a pytorch loss function or None @@ -89,6 +94,8 @@ class Trainer(object): self._summary_writer = SummaryWriter(self.pickle_path + 'tensorboard_logs') self._graph_summaried = False self._best_accuracy = 0.0 + self.eval_sort_key = default_args['eval_sort_key'] + self.validator = None def train(self, network, train_data, dev_data=None): """General Training Procedure @@ -108,8 +115,9 @@ class Trainer(object): if self.validate: default_valid_args = {"batch_size": self.batch_size, "pickle_path": self.pickle_path, "use_cuda": self.use_cuda, "evaluator": self._evaluator} - validator = self._create_validator(default_valid_args) - logger.info("validator defined as {}".format(str(validator))) + if self.validator is None: + self.validator = self._create_validator(default_valid_args) + logger.info("validator defined as {}".format(str(self.validator))) # optimizer and loss self.define_optimizer() @@ -117,29 +125,31 @@ class Trainer(object): self.define_loss() logger.info("loss function defined as {}".format(str(self._loss_func))) + # turn on network training mode + self.mode(network, is_test=False) + # main training procedure start = time.time() - logger.info("training epochs started") - for epoch in range(1, self.n_epochs + 1): + self.start_time = str(start) + + logger.info("training epochs started " + self.start_time) + epoch, iters = 1, 0 + while(1): + if self.n_epochs != -1 and epoch > self.n_epochs: + break logger.info("training epoch {}".format(epoch)) - # turn on network training mode - self.mode(network, is_test=False) # prepare mini-batch iterator data_iterator = Batch(train_data, batch_size=self.batch_size, sampler=RandomSampler(), use_cuda=self.use_cuda) logger.info("prepared data iterator") # one forward and backward pass - self._train_step(data_iterator, network, start=start, n_print=self.print_every_step, epoch=epoch) + iters += self._train_step(data_iterator, network, start=start, n_print=self.print_every_step, epoch=epoch, step=iters, dev_data=dev_data) # validation if self.validate: - if dev_data is None: - raise RuntimeError( - "self.validate is True in trainer, but dev_data is None. Please provide the validation data.") - logger.info("validation started") - validator.test(network, dev_data) + self.valid_model() def _train_step(self, data_iterator, network, **kwargs): """Training process in one epoch. @@ -149,7 +159,8 @@ class Trainer(object): - start: time.time(), the starting time of this step. - epoch: int, """ - step = 0 + step = kwargs['step'] + dev_data = kwargs['dev_data'] for batch_x, batch_y in data_iterator: prediction = self.data_forward(network, batch_x) @@ -166,7 +177,21 @@ class Trainer(object): kwargs["epoch"], step, loss.data, diff) print(print_output) logger.info(print_output) + if self.validate and self.valid_step > 0 and step > 0 and step % self.valid_step == 0: + self.valid_model() step += 1 + return step + + def valid_model(self): + if dev_data is None: + raise RuntimeError( + "self.validate is True in trainer, but dev_data is None. Please provide the validation data.") + logger.info("validation started") + res = self.validator.test(network, dev_data) + if self.save_best_dev and self.best_eval_result(res): + logger.info('save best result! {}'.format(res)) + self.save_model(self._model, 'best_model_'+self.start_time) + return res def mode(self, model, is_test=False): """Train mode or Test mode. This is for PyTorch currently. @@ -180,11 +205,17 @@ class Trainer(object): else: model.train() - def define_optimizer(self): + def define_optimizer(self, optim=None): """Define framework-specific optimizer specified by the models. """ - self._optimizer = self._optimizer_proto.construct_from_pytorch(self._model.parameters()) + if optim is not None: + # optimizer constructed by user + self._optimizer = optim + elif self._optimizer is None: + # optimizer constructed by proto + self._optimizer = self._optimizer_proto.construct_from_pytorch(self._model.parameters()) + return self._optimizer def update(self): """Perform weight update on a model. @@ -217,6 +248,8 @@ class Trainer(object): :param truth: ground truth label vector :return: a scalar """ + if isinstance(predict, dict) and isinstance(truth, dict): + return self._loss_func(**predict, **truth) if len(truth) > 1: raise NotImplementedError("Not ready to handle multi-labels.") truth = list(truth.values())[0] if len(truth) > 0 else None @@ -241,13 +274,27 @@ class Trainer(object): raise ValueError("Please specify a loss function.") logger.info("The model didn't define loss, use Trainer's loss.") - def best_eval_result(self, validator): + def best_eval_result(self, metrics): """Check if the current epoch yields better validation results. :param validator: a Tester instance :return: bool, True means current results on dev set is the best. """ - loss, accuracy = validator.metrics + if isinstance(metrics, tuple): + loss, metrics = metrics + else: + metrics = validator.metrics + + if isinstance(metrics, dict): + if len(metrics) == 1: + accuracy = list(metrics.values())[0] + elif self.eval_sort_key is None: + raise ValueError('dict format metrics should provide sort key for eval best result') + else: + accuracy = metrics[self.eval_sort_key] + else: + accuracy = metrics + if accuracy > self._best_accuracy: self._best_accuracy = accuracy return True @@ -268,6 +315,8 @@ class Trainer(object): def _create_validator(self, valid_args): raise NotImplementedError + def set_validator(self, validor): + self.validator = validor class SeqLabelTrainer(Trainer): """Trainer for Sequence Labeling diff --git a/fastNLP/models/biaffine_parser.py b/fastNLP/models/biaffine_parser.py index a5461ee8..4561dbd2 100644 --- a/fastNLP/models/biaffine_parser.py +++ b/fastNLP/models/biaffine_parser.py @@ -243,6 +243,9 @@ class BiaffineParser(GraphParser): self.normal_dropout = nn.Dropout(p=dropout) self.use_greedy_infer = use_greedy_infer initial_parameter(self) + self.word_norm = nn.LayerNorm(word_emb_dim) + self.pos_norm = nn.LayerNorm(pos_emb_dim) + self.lstm_norm = nn.LayerNorm(rnn_out_size) def forward(self, word_seq, pos_seq, word_seq_origin_len, gold_heads=None, **_): """ @@ -266,10 +269,12 @@ class BiaffineParser(GraphParser): word = self.normal_dropout(self.word_embedding(word_seq)) # [N,L] -> [N,L,C_0] pos = self.normal_dropout(self.pos_embedding(pos_seq)) # [N,L] -> [N,L,C_1] + word, pos = self.word_norm(word), self.pos_norm(pos) x = torch.cat([word, pos], dim=2) # -> [N,L,C] # lstm, extract features feat, _ = self.lstm(x) # -> [N,L,C] + feat = self.lstm_norm(feat) # for arc biaffine # mlp, reduce dim @@ -292,6 +297,7 @@ class BiaffineParser(GraphParser): heads = self._mst_decoder(arc_pred, seq_mask) head_pred = heads else: + assert self.training # must be training mode head_pred = None heads = gold_heads @@ -331,40 +337,4 @@ class BiaffineParser(GraphParser): label_nll = -(label_loss*float_mask).sum() / length return arc_nll + label_nll - def evaluate(self, arc_pred, label_pred, head_indices, head_labels, seq_mask, **kwargs): - """ - Evaluate the performance of prediction. - - :return dict: performance results. - head_pred_corrct: number of correct predicted heads. - label_pred_correct: number of correct predicted labels. - total_tokens: number of predicted tokens - """ - if 'head_pred' in kwargs: - head_pred = kwargs['head_pred'] - elif self.use_greedy_infer: - head_pred = self._greedy_decoder(arc_pred, seq_mask) - else: - head_pred = self._mst_decoder(arc_pred, seq_mask) - - head_pred_correct = (head_pred == head_indices).long() * seq_mask - _, label_preds = torch.max(label_pred, dim=2) - label_pred_correct = (label_preds == head_labels).long() * head_pred_correct - return {"head_pred_correct": head_pred_correct.sum(dim=1), - "label_pred_correct": label_pred_correct.sum(dim=1), - "total_tokens": seq_mask.sum(dim=1)} - - def metrics(self, head_pred_correct, label_pred_correct, total_tokens, **_): - """ - Compute the metrics of model - - :param head_pred_corrct: number of correct predicted heads. - :param label_pred_correct: number of correct predicted labels. - :param total_tokens: number of predicted tokens - :return dict: the metrics results - UAS: the head predicted accuracy - LAS: the label predicted accuracy - """ - return {"UAS": head_pred_correct.sum().float() / total_tokens.sum().float() * 100, - "LAS": label_pred_correct.sum().float() / total_tokens.sum().float() * 100} diff --git a/reproduction/Biaffine_parser/cfg.cfg b/reproduction/Biaffine_parser/cfg.cfg index 84e0f288..3adb6937 100644 --- a/reproduction/Biaffine_parser/cfg.cfg +++ b/reproduction/Biaffine_parser/cfg.cfg @@ -1,23 +1,25 @@ [train] epochs = -1 +<<<<<<< HEAD batch_size = 16 +======= +batch_size = 32 +>>>>>>> update biaffine pickle_path = "./save/" validate = true -save_best_dev = false +save_best_dev = true +eval_sort_key = "UAS" use_cuda = true model_saved_path = "./save/" -task = "parse" - [test] save_output = true validate_in_training = true save_dev_input = false save_loss = true -batch_size = 16 +batch_size = 64 pickle_path = "./save/" use_cuda = true -task = "parse" [model] word_vocab_size = -1 diff --git a/reproduction/Biaffine_parser/run.py b/reproduction/Biaffine_parser/run.py index 9404d195..5bab554a 100644 --- a/reproduction/Biaffine_parser/run.py +++ b/reproduction/Biaffine_parser/run.py @@ -8,12 +8,14 @@ import math import torch from fastNLP.core.trainer import Trainer +from fastNLP.core.metrics import Evaluator from fastNLP.core.instance import Instance from fastNLP.core.vocabulary import Vocabulary from fastNLP.core.dataset import DataSet from fastNLP.core.batch import Batch from fastNLP.core.sampler import SequentialSampler from fastNLP.core.field import TextField, SeqLabelField +from fastNLP.core.preprocess import load_pickle from fastNLP.core.tester import Tester from fastNLP.loader.config_loader import ConfigLoader, ConfigSection from fastNLP.loader.model_loader import ModelLoader @@ -111,9 +113,10 @@ class CTBDataLoader(object): # emb_file_name = '/home/yfshao/glove.6B.100d.txt' # loader = ConlluDataLoader() -datadir = "/home/yfshao/parser-data" +datadir = '/home/yfshao/workdir/parser-data/' train_data_name = "train_ctb5.txt" dev_data_name = "dev_ctb5.txt" +test_data_name = "test_ctb5.txt" emb_file_name = "/home/yfshao/parser-data/word_OOVthr_30_100v.txt" loader = CTBDataLoader() @@ -148,37 +151,33 @@ def load_data(dirpath): datas[name] = _pickle.load(f) return datas -class MyTester(object): - def __init__(self, batch_size, use_cuda=False, **kwagrs): - self.batch_size = batch_size - self.use_cuda = use_cuda - - def test(self, model, dataset): - self.model = model.cuda() if self.use_cuda else model - self.model.eval() - batchiter = Batch(dataset, self.batch_size, SequentialSampler(), self.use_cuda) - eval_res = defaultdict(list) - i = 0 - for batch_x, batch_y in batchiter: - with torch.no_grad(): - pred_y = self.model(**batch_x) - eval_one = self.model.evaluate(**pred_y, **batch_y) - i += self.batch_size - for eval_name, tensor in eval_one.items(): - eval_res[eval_name].append(tensor) - tmp = {} - for eval_name, tensorlist in eval_res.items(): - tmp[eval_name] = torch.cat(tensorlist, dim=0) - - self.res = self.model.metrics(**tmp) - print(self.show_metrics()) - - def show_metrics(self): - s = "" - for name, val in self.res.items(): - s += '{}: {:.2f}\t'.format(name, val) - return s +class ParserEvaluator(Evaluator): + def __init__(self): + super(ParserEvaluator, self).__init__() + def __call__(self, predict_list, truth_list): + head_all, label_all, total_all = 0, 0, 0 + for pred, truth in zip(predict_list, truth_list): + head, label, total = self.evaluate(**pred, **truth) + head_all += head + label_all += label + total_all += total + + return {'UAS': head_all*1.0 / total_all, 'LAS': label_all*1.0 / total_all} + + def evaluate(self, head_pred, label_pred, head_indices, head_labels, seq_mask, **_): + """ + Evaluate the performance of prediction. + + :return : performance results. + head_pred_corrct: number of correct predicted heads. + label_pred_correct: number of correct predicted labels. + total_tokens: number of predicted tokens + """ + head_pred_correct = (head_pred == head_indices).long() * seq_mask + _, label_preds = torch.max(label_pred, dim=2) + label_pred_correct = (label_preds == head_labels).long() * head_pred_correct + return head_pred_correct.sum().item(), label_pred_correct.sum().item(), seq_mask.sum().item() try: data_dict = load_data(processed_datadir) @@ -196,6 +195,7 @@ except Exception as _: tag_v = Vocabulary(need_default=False) train_data = loader.load(os.path.join(datadir, train_data_name)) dev_data = loader.load(os.path.join(datadir, dev_data_name)) + test_data = loader.load(os.path.join(datadir, test_data_name)) train_data.update_vocab(word_seq=word_v, pos_seq=pos_v, head_labels=tag_v) save_data(processed_datadir, word_v=word_v, pos_v=pos_v, tag_v=tag_v, train_data=train_data, dev_data=dev_data) @@ -207,8 +207,6 @@ dev_data.set_origin_len("word_seq") print(train_data[:3]) print(len(train_data)) print(len(dev_data)) -ep = train_args['epochs'] -train_args['epochs'] = math.ceil(50000.0 / len(train_data) * train_args['batch_size']) if ep <= 0 else ep model_args['word_vocab_size'] = len(word_v) model_args['pos_vocab_size'] = len(pos_v) model_args['num_label'] = len(tag_v) @@ -220,7 +218,7 @@ def train(): def _define_optim(obj): obj._optimizer = torch.optim.Adam(obj._model.parameters(), **optim_args.data) - obj._scheduler = torch.optim.lr_scheduler.LambdaLR(obj._optimizer, lambda ep: .75 ** (ep / 5e4)) + obj._scheduler = torch.optim.lr_scheduler.LambdaLR(obj._optimizer, lambda ep: max(.75 ** (ep / 5e4), 0.05)) def _update(obj): obj._scheduler.step() @@ -228,8 +226,7 @@ def train(): trainer.define_optimizer = lambda: _define_optim(trainer) trainer.update = lambda: _update(trainer) - trainer.get_loss = lambda predict, truth: trainer._loss_func(**predict, **truth) - trainer._create_validator = lambda x: MyTester(**test_args.data) + trainer.set_validator(Tester(**test_args.data, evaluator=ParserEvaluator())) # Model model = BiaffineParser(**model_args.data) @@ -238,6 +235,7 @@ def train(): word_v.unknown_label = "" embed, _ = EmbedLoader.load_embedding(model_args['word_emb_dim'], emb_file_name, 'glove', word_v, os.path.join(processed_datadir, 'word_emb.pkl')) model.word_embedding = torch.nn.Embedding.from_pretrained(embed, freeze=False) + model.word_embedding.padding_idx = word_v.padding_idx model.word_embedding.weight.data[word_v.padding_idx].fill_(0) model.pos_embedding.padding_idx = pos_v.padding_idx @@ -262,7 +260,7 @@ def train(): def test(): # Tester - tester = MyTester(**test_args.data) + tester = Tester(**test_args.data, evaluator=ParserEvaluator()) # Model model = BiaffineParser(**model_args.data) @@ -275,9 +273,10 @@ def test(): raise # Start training + print("Testing Dev data") tester.test(model, dev_data) - print(tester.show_metrics()) - print("Testing finished!") + print("Testing Test data") + tester.test(model, test_data)