From 87a19b91cbd52e1c422a3b3a23b6cfe33f9e1554 Mon Sep 17 00:00:00 2001 From: ChenXin Date: Fri, 28 Feb 2020 00:18:08 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E4=BA=86=20tutorial=5F7=20?= =?UTF-8?q?=E5=B9=B6=E6=8F=90=E4=BE=9B=E4=BA=86=20ipynb?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/tutorials/tutorial_7_metrics.rst | 75 +- tutorials/tutorial_7_metrics.ipynb | 1200 ++++++++++++++++++ 2 files changed, 1241 insertions(+), 34 deletions(-) create mode 100644 tutorials/tutorial_7_metrics.ipynb diff --git a/docs/source/tutorials/tutorial_7_metrics.rst b/docs/source/tutorials/tutorial_7_metrics.rst index 0b4f86c8..bb17292c 100644 --- a/docs/source/tutorials/tutorial_7_metrics.rst +++ b/docs/source/tutorials/tutorial_7_metrics.rst @@ -3,14 +3,13 @@ =============================== 在进行训练时，fastNLP提供了各种各样的 :mod:`~fastNLP.core.metrics` 。 -如 :doc:`/user/quickstart` 中所介绍的，:class:`~fastNLP.AccuracyMetric` 类的对象被直接传到 :class:`~fastNLP.Trainer` 中用于训练 +如前面的教程中所介绍，:class:`~fastNLP.AccuracyMetric` 类的对象被直接传到 :class:`~fastNLP.Trainer` 中用于训练 .. code-block:: python - from fastNLP import Trainer, CrossEntropyLoss, AccuracyMetric - - trainer = Trainer(model=model, train_data=train_data, dev_data=dev_data, - loss=CrossEntropyLoss(), metrics=AccuracyMetric()) + trainer = Trainer(train_data=train_data, model=model, loss=loss, + optimizer=optimizer, batch_size=32, dev_data=dev_data, + metrics=metric, device=device) trainer.train() 除了 :class:`~fastNLP.AccuracyMetric` 之外，:class:`~fastNLP.SpanFPreRecMetric` 也是一种非常见的评价指标， @@ -40,7 +39,7 @@ get_metric(xxx) 当所有数据处理完毕时调用该方法，它将根据 evaluate函数累计的评价指标统计量来计算最终的评价结果 -以分类问题中，Accuracy计算为例，假设model的forward返回dict中包含 `pred` 这个key, 并且该key需要用于Accuracy:: +以分类问题中，accuracy 计算为例，假设 model 的 `forward` 返回 dict 中包含 `pred` 这个 key , 并且该 key 需要用于 accuracy:: class Model(nn.Module): def __init__(xxx): @@ -49,58 +48,67 @@ # do something return {'pred': pred, 'other_keys':xxx} # pred's shape: batch_size x num_classes -假设dataset中 `label` 这个field是需要预测的值，并且该field被设置为了target -对应的AccMetric可以按如下的定义, version1, 只使用这一次:: +假设dataset中 `target` 这个 field 是需要预测的值，并且该 field 被设置为了 target 对应的 `AccMetric` 可以按如下的定义( Version 1, 只使用这一次):: + + from fastNLP import MetricBase class AccMetric(MetricBase): + def __init__(self): super().__init__() - # 根据你的情况自定义指标 - self.corr_num = 0 self.total = 0 + self.acc_count = 0 - def evaluate(self, label, pred): # 这里的名称需要和dataset中target field与model返回的key是一样的，不然找不到对应的value + # evaluate的参数需要和DataSet 中 field 名以及模型输出的结果 field 名一致，不然找不到对应的value + # pred, target 的参数是 fastNLP 的默认配置 + def evaluate(self, pred, target): # dev或test时，每个batch结束会调用一次该方法，需要实现如何根据每个batch累加metric - self.total += label.size(0) - self.corr_num += label.eq(pred).sum().item() + self.total += target.size(0) + self.acc_count += target.eq(pred).sum().item() def get_metric(self, reset=True): # 在这里定义如何计算metric - acc = self.corr_num/self.total + acc = self.acc_count/self.total if reset: # 是否清零以便重新计算 - self.corr_num = 0 + self.acc_count = 0 self.total = 0 - return {'acc': acc} # 需要返回一个dict，key为该metric的名称，该名称会显示到Trainer的progress bar中 + return {'acc': acc} + # 需要返回一个dict，key为该metric的名称，该名称会显示到Trainer的progress bar中 -version2，如果需要复用Metric，比如下一次使用AccMetric时，dataset中目标field不叫label而叫y，或者model的输出不是pred:: +如果需要复用 metric，比如下一次使用 `AccMetric` 时，dataset中目标field不叫 `target` 而叫 `y` ，或者model的输出不是 `pred` (Version 2):: class AccMetric(MetricBase): - def __init__(self, label=None, pred=None): - # 假设在另一场景使用时，目标field叫y，model给出的key为pred_y。则只需要在初始化AccMetric时， - # acc_metric = AccMetric(label='y', pred='pred_y')即可。 - # 当初始化为acc_metric = AccMetric()，即label=None, pred=None, fastNLP会直接使用'label', 'pred'作为key去索取对 - # 应的的值 + def __init__(self, pred=None, target=None): + """ + 假设在另一场景使用时，目标field叫y，model给出的key为pred_y。则只需要在初始化AccMetric时， + acc_metric = AccMetric(pred='pred_y', target='y')即可。 + 当初始化为acc_metric = AccMetric() 时，fastNLP会直接使用 'pred', 'target' 作为key去索取对应的的值 + """ + super().__init__() - self._init_param_map(label=label, pred=pred) # 该方法会注册label和pred. 仅需要注册evaluate()方法会用到的参数名即可 - # 如果没有注册该则效果与version1就是一样的 + + # 如果没有注册该则效果与 Version 1 就是一样的 + self._init_param_map(pred=pred, target=target) # 该方法会注册label和pred. 仅需要注册evaluate()方法会用到的参数名即可 # 根据你的情况自定义指标 - self.corr_num = 0 self.total = 0 + self.acc_count = 0 - def evaluate(self, label, pred): # 这里的参数名称需要和self._init_param_map()注册时一致。 + # evaluate的参数需要和DataSet 中 field 名以及模型输出的结果 field 名一致，不然找不到对应的value + # pred, target 的参数是 fastNLP 的默认配置 + def evaluate(self, pred, target): # dev或test时，每个batch结束会调用一次该方法，需要实现如何根据每个batch累加metric - self.total += label.size(0) - self.corr_num += label.eq(pred).sum().item() + self.total += target.size(0) + self.acc_count += target.eq(pred).sum().item() def get_metric(self, reset=True): # 在这里定义如何计算metric - acc = self.corr_num/self.total + acc = self.acc_count/self.total if reset: # 是否清零以便重新计算 - self.corr_num = 0 + self.acc_count = 0 self.total = 0 - return {'acc': acc} # 需要返回一个dict，key为该metric的名称，该名称会显示到Trainer的progress bar中 - + return {'acc': acc} + # 需要返回一个dict，key为该metric的名称，该名称会显示到Trainer的progress bar中 ``MetricBase`` 将会在输入的字典 ``pred_dict`` 和 ``target_dict`` 中进行检查. ``pred_dict`` 是模型当中 ``forward()`` 函数或者 ``predict()`` 函数的返回值. @@ -108,14 +116,13 @@ version2，如果需要复用Metric，比如下一次使用AccMetric时，datase ``MetricBase`` 会进行以下的类型检测: -1. self.evaluate当中是否有varargs, 这是不支持的. +1. self.evaluate当中是否有 varargs, 这是不支持的. 2. self.evaluate当中所需要的参数是否既不在 ``pred_dict`` 也不在 ``target_dict`` . 3. self.evaluate当中所需要的参数是否既在 ``pred_dict`` 也在 ``target_dict`` . 除此以外，在参数被传入self.evaluate以前，这个函数会检测 ``pred_dict`` 和 ``target_dict`` 当中没有被用到的参数 如果kwargs是self.evaluate的参数，则不会检测 - self.evaluate将计算一个批次(batch)的评价指标，并累计。 没有返回值 self.get_metric将统计当前的评价指标并返回评价结果, 返回值需要是一个dict, key是指标名称，value是指标的值 diff --git a/tutorials/tutorial_7_metrics.ipynb b/tutorials/tutorial_7_metrics.ipynb new file mode 100644 index 00000000..e6780587 --- /dev/null +++ b/tutorials/tutorial_7_metrics.ipynb @@ -0,0 +1,1200 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 使用Metric快速评测你的模型\n", + "\n", + "和上一篇教程一样的实验准备代码" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/remote-home/ynzheng/anaconda3/envs/now/lib/python3.8/site-packages/FastNLP-0.5.0-py3.8.egg/fastNLP/io/loader/classification.py:340: UserWarning: SST2's test file has no target.\n" + ] + } + ], + "source": [ + "from fastNLP.io import SST2Pipe\n", + "from fastNLP import Trainer, CrossEntropyLoss, AccuracyMetric\n", + "from fastNLP.models import CNNText\n", + "from fastNLP import CrossEntropyLoss\n", + "import torch\n", + "from torch.optim import Adam\n", + "from fastNLP import AccuracyMetric\n", + "\n", + "databundle = SST2Pipe().process_from_file()\n", + "vocab = databundle.get_vocab('words')\n", + "train_data = databundle.get_dataset('train')[:5000]\n", + "train_data, test_data = train_data.split(0.015)\n", + "dev_data = databundle.get_dataset('dev')\n", + "\n", + "model = CNNText((len(vocab),100), num_classes=2, dropout=0.1)\n", + "loss = CrossEntropyLoss()\n", + "metric = AccuracyMetric()\n", + "optimizer = Adam(model.parameters(), lr=0.001)\n", + "device = 0 if torch.cuda.is_available() else 'cpu'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "进行训练时，fastNLP提供了各种各样的 metrics 。 如前面的教程中所介绍，AccuracyMetric 类的对象被直接传到 Trainer 中用于训练" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "input fields after batch(if batch size is 2):\n", + "\twords: (1)type:torch.Tensor (2)dtype:torch.int64, (3)shape:torch.Size([2, 13]) \n", + "\tseq_len: (1)type:torch.Tensor (2)dtype:torch.int64, (3)shape:torch.Size([2]) \n", + "target fields after batch(if batch size is 2):\n", + "\ttarget: (1)type:torch.Tensor (2)dtype:torch.int64, (3)shape:torch.Size([2]) \n", + "\n", + "training epochs started 2020-02-28-00-11-51\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=1540.0), HTML(value='')), layout=Layout(d…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=28.0), HTML(value='')), layout=Layout(dis…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Evaluate data in 0.16 seconds!\n", + "\r", + "Evaluation on dev at Epoch 1/10. Step:154/1540: \n", + "\r", + "AccuracyMetric: acc=0.722477\n", + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=28.0), HTML(value='')), layout=Layout(dis…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Evaluate data in 0.36 seconds!\n", + "\r", + "Evaluation on dev at Epoch 2/10. Step:308/1540: \n", + "\r", + "AccuracyMetric: acc=0.762615\n", + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=28.0), HTML(value='')), layout=Layout(dis…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Evaluate data in 0.16 seconds!\n", + "\r", + "Evaluation on dev at Epoch 3/10. Step:462/1540: \n", + "\r", + "AccuracyMetric: acc=0.771789\n", + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=28.0), HTML(value='')), layout=Layout(dis…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Evaluate data in 0.44 seconds!\n", + "\r", + "Evaluation on dev at Epoch 4/10. Step:616/1540: \n", + "\r", + "AccuracyMetric: acc=0.759174\n", + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=28.0), HTML(value='')), layout=Layout(dis…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Evaluate data in 0.29 seconds!\n", + "\r", + "Evaluation on dev at Epoch 5/10. Step:770/1540: \n", + "\r", + "AccuracyMetric: acc=0.75344\n", + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=28.0), HTML(value='')), layout=Layout(dis…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Evaluate data in 0.33 seconds!\n", + "\r", + "Evaluation on dev at Epoch 6/10. Step:924/1540: \n", + "\r", + "AccuracyMetric: acc=0.75\n", + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=28.0), HTML(value='')), layout=Layout(dis…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Evaluate data in 0.19 seconds!\n", + "\r", + "Evaluation on dev at Epoch 7/10. Step:1078/1540: \n", + "\r", + "AccuracyMetric: acc=0.741972\n", + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=28.0), HTML(value='')), layout=Layout(dis…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Evaluate data in 0.49 seconds!\n", + "\r", + "Evaluation on dev at Epoch 8/10. Step:1232/1540: \n", + "\r", + "AccuracyMetric: acc=0.740826\n", + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=28.0), HTML(value='')), layout=Layout(dis…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Evaluate data in 0.15 seconds!\n", + "\r", + "Evaluation on dev at Epoch 9/10. Step:1386/1540: \n", + "\r", + "AccuracyMetric: acc=0.75\n", + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=28.0), HTML(value='')), layout=Layout(dis…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Evaluate data in 0.16 seconds!\n", + "\r", + "Evaluation on dev at Epoch 10/10. Step:1540/1540: \n", + "\r", + "AccuracyMetric: acc=0.752294\n", + "\n", + "\r\n", + "In Epoch:3/Step:462, got best dev performance:\n", + "AccuracyMetric: acc=0.771789\n", + "Reloaded the best model.\n" + ] + }, + { + "data": { + "text/plain": [ + "{'best_eval': {'AccuracyMetric': {'acc': 0.771789}},\n", + " 'best_epoch': 3,\n", + " 'best_step': 462,\n", + " 'seconds': 30.04}" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "trainer = Trainer(train_data=train_data, model=model, loss=loss,\n", + " optimizer=optimizer, batch_size=32, dev_data=dev_data,\n", + " metrics=metric, device=device)\n", + "trainer.train()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "除了 AccuracyMetric 之外，SpanFPreRecMetric 也是一种非常见的评价指标， 例如在序列标注问题中，常以span的方式计算 F-measure, precision, recall。\n", + "\n", + "另外，fastNLP 还实现了用于抽取式QA（如SQuAD）的metric ExtractiveQAMetric。 用户可以参考下面这个表格。\n", + "\n", + "| 名称 | 介绍 |\n", + "| -------------------- | ------------------------------------------------- |\n", + "| `MetricBase` | 自定义metrics需继承的基类 |\n", + "| `AccuracyMetric` | 简单的正确率metric |\n", + "| `SpanFPreRecMetric` | 同时计算 F-measure, precision, recall 值的 metric |\n", + "| `ExtractiveQAMetric` | 用于抽取式QA任务 的metric |\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 定义自己的metrics\n", + "\n", + "在定义自己的metrics类时需继承 fastNLP 的 MetricBase, 并覆盖写入 evaluate 和 get_metric 方法。\n", + "\n", + "- evaluate(xxx) 中传入一个批次的数据，将针对一个批次的预测结果做评价指标的累计\n", + "\n", + "- get_metric(xxx) 当所有数据处理完毕时调用该方法，它将根据 evaluate函数累计的评价指标统计量来计算最终的评价结果\n", + "\n", + "以分类问题中，Accuracy计算为例，假设model的forward返回dict中包含 pred 这个key, 并且该key需要用于Accuracy:\n", + "\n", + "```python\n", + "class Model(nn.Module):\n", + " def __init__(xxx):\n", + " # do something\n", + " def forward(self, xxx):\n", + " # do something\n", + " return {'pred': pred, 'other_keys':xxx} # pred's shape: batch_size x num_classes\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Version 1\n", + "\n", + "假设dataset中 `target` 这个 field 是需要预测的值，并且该 field 被设置为了 target 对应的 `AccMetric` 可以按如下的定义" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "from fastNLP import MetricBase\n", + "\n", + "class AccMetric(MetricBase):\n", + "\n", + " def __init__(self):\n", + " super().__init__()\n", + " # 根据你的情况自定义指标\n", + " self.total = 0\n", + " self.acc_count = 0\n", + "\n", + " # evaluate的参数需要和DataSet 中 field 名以及模型输出的结果 field 名一致，不然找不到对应的value\n", + " # pred, target 的参数是 fastNLP 的默认配置\n", + " def evaluate(self, pred, target):\n", + " # dev或test时，每个batch结束会调用一次该方法，需要实现如何根据每个batch累加metric\n", + " self.total += target.size(0)\n", + " self.acc_count += target.eq(pred).sum().item()\n", + "\n", + " def get_metric(self, reset=True): # 在这里定义如何计算metric\n", + " acc = self.acc_count/self.total\n", + " if reset: # 是否清零以便重新计算\n", + " self.acc_count = 0\n", + " self.total = 0\n", + " return {'acc': acc}\n", + " # 需要返回一个dict，key为该metric的名称，该名称会显示到Trainer的progress bar中" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "input fields after batch(if batch size is 2):\n", + "\twords: (1)type:torch.Tensor (2)dtype:torch.int64, (3)shape:torch.Size([2, 13]) \n", + "\tseq_len: (1)type:torch.Tensor (2)dtype:torch.int64, (3)shape:torch.Size([2]) \n", + "target fields after batch(if batch size is 2):\n", + "\ttarget: (1)type:torch.Tensor (2)dtype:torch.int64, (3)shape:torch.Size([2]) \n", + "\n", + "training epochs started 2020-02-28-00-12-21\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=1540.0), HTML(value='')), layout=Layout(d…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=28.0), HTML(value='')), layout=Layout(dis…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Evaluate data in 0.33 seconds!\n", + "\r", + "Evaluation on dev at Epoch 1/10. Step:154/1540: \n", + "\r", + "AccMetric: acc=0.7419724770642202\n", + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=28.0), HTML(value='')), layout=Layout(dis…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Evaluate data in 0.19 seconds!\n", + "\r", + "Evaluation on dev at Epoch 2/10. Step:308/1540: \n", + "\r", + "AccMetric: acc=0.7660550458715596\n", + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=28.0), HTML(value='')), layout=Layout(dis…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Evaluate data in 0.27 seconds!\n", + "\r", + "Evaluation on dev at Epoch 3/10. Step:462/1540: \n", + "\r", + "AccMetric: acc=0.75\n", + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=28.0), HTML(value='')), layout=Layout(dis…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Evaluate data in 0.24 seconds!\n", + "\r", + "Evaluation on dev at Epoch 4/10. Step:616/1540: \n", + "\r", + "AccMetric: acc=0.7534403669724771\n", + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=28.0), HTML(value='')), layout=Layout(dis…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Evaluate data in 0.29 seconds!\n", + "\r", + "Evaluation on dev at Epoch 5/10. Step:770/1540: \n", + "\r", + "AccMetric: acc=0.7488532110091743\n", + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=28.0), HTML(value='')), layout=Layout(dis…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Evaluate data in 0.14 seconds!\n", + "\r", + "Evaluation on dev at Epoch 6/10. Step:924/1540: \n", + "\r", + "AccMetric: acc=0.7488532110091743\n", + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=28.0), HTML(value='')), layout=Layout(dis…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Evaluate data in 0.27 seconds!\n", + "\r", + "Evaluation on dev at Epoch 7/10. Step:1078/1540: \n", + "\r", + "AccMetric: acc=0.7568807339449541\n", + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=28.0), HTML(value='')), layout=Layout(dis…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Evaluate data in 0.42 seconds!\n", + "\r", + "Evaluation on dev at Epoch 8/10. Step:1232/1540: \n", + "\r", + "AccMetric: acc=0.7488532110091743\n", + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=28.0), HTML(value='')), layout=Layout(dis…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Evaluate data in 0.16 seconds!\n", + "\r", + "Evaluation on dev at Epoch 9/10. Step:1386/1540: \n", + "\r", + "AccMetric: acc=0.7408256880733946\n", + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=28.0), HTML(value='')), layout=Layout(dis…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Evaluate data in 0.28 seconds!\n", + "\r", + "Evaluation on dev at Epoch 10/10. Step:1540/1540: \n", + "\r", + "AccMetric: acc=0.7408256880733946\n", + "\n", + "\r\n", + "In Epoch:2/Step:308, got best dev performance:\n", + "AccMetric: acc=0.7660550458715596\n", + "Reloaded the best model.\n" + ] + }, + { + "data": { + "text/plain": [ + "{'best_eval': {'AccMetric': {'acc': 0.7660550458715596}},\n", + " 'best_epoch': 2,\n", + " 'best_step': 308,\n", + " 'seconds': 29.74}" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "trainer = Trainer(train_data=train_data, model=model, loss=loss,\n", + " optimizer=optimizer, batch_size=32, dev_data=dev_data,\n", + " metrics=AccMetric(), device=device)\n", + "trainer.train()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Version 2\n", + "\n", + "如果需要复用 metric，比如下一次使用 `AccMetric` 时，dataset中目标field不叫 `target` 而叫 `y` ，或者model的输出不是 `pred`\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "class AccMetric(MetricBase):\n", + " def __init__(self, pred=None, target=None):\n", + " \"\"\"\n", + " 假设在另一场景使用时，目标field叫y，model给出的key为pred_y。则只需要在初始化AccMetric时，\n", + " acc_metric = AccMetric(pred='pred_y', target='y')即可。\n", + " 当初始化为acc_metric = AccMetric() 时，fastNLP会直接使用 'pred', 'target' 作为key去索取对应的的值\n", + " \"\"\"\n", + "\n", + " super().__init__()\n", + "\n", + " # 如果没有注册该则效果与 Version 1 就是一样的\n", + " self._init_param_map(pred=pred, target=target) # 该方法会注册label和pred. 仅需要注册evaluate()方法会用到的参数名即可\n", + "\n", + " # 根据你的情况自定义指标\n", + " self.total = 0\n", + " self.acc_count = 0\n", + "\n", + " # evaluate的参数需要和DataSet 中 field 名以及模型输出的结果 field 名一致，不然找不到对应的value\n", + " # pred, target 的参数是 fastNLP 的默认配置\n", + " def evaluate(self, pred, target):\n", + " # dev或test时，每个batch结束会调用一次该方法，需要实现如何根据每个batch累加metric\n", + " self.total += target.size(0)\n", + " self.acc_count += target.eq(pred).sum().item()\n", + "\n", + " def get_metric(self, reset=True): # 在这里定义如何计算metric\n", + " acc = self.acc_count/self.total\n", + " if reset: # 是否清零以便重新计算\n", + " self.acc_count = 0\n", + " self.total = 0\n", + " return {'acc': acc}\n", + " # 需要返回一个dict，key为该metric的名称，该名称会显示到Trainer的progress bar中" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "input fields after batch(if batch size is 2):\n", + "\twords: (1)type:torch.Tensor (2)dtype:torch.int64, (3)shape:torch.Size([2, 13]) \n", + "\tseq_len: (1)type:torch.Tensor (2)dtype:torch.int64, (3)shape:torch.Size([2]) \n", + "target fields after batch(if batch size is 2):\n", + "\ttarget: (1)type:torch.Tensor (2)dtype:torch.int64, (3)shape:torch.Size([2]) \n", + "\n", + "training epochs started 2020-02-28-00-12-51\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=1540.0), HTML(value='')), layout=Layout(d…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=28.0), HTML(value='')), layout=Layout(dis…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Evaluate data in 0.24 seconds!\n", + "\r", + "Evaluation on dev at Epoch 1/10. Step:154/1540: \n", + "\r", + "AccMetric: acc=0.7545871559633027\n", + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=28.0), HTML(value='')), layout=Layout(dis…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Evaluate data in 0.24 seconds!\n", + "\r", + "Evaluation on dev at Epoch 2/10. Step:308/1540: \n", + "\r", + "AccMetric: acc=0.7534403669724771\n", + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=28.0), HTML(value='')), layout=Layout(dis…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Evaluate data in 0.18 seconds!\n", + "\r", + "Evaluation on dev at Epoch 3/10. Step:462/1540: \n", + "\r", + "AccMetric: acc=0.7557339449541285\n", + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=28.0), HTML(value='')), layout=Layout(dis…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Evaluate data in 0.11 seconds!\n", + "\r", + "Evaluation on dev at Epoch 4/10. Step:616/1540: \n", + "\r", + "AccMetric: acc=0.7511467889908257\n", + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=28.0), HTML(value='')), layout=Layout(dis…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Evaluate data in 0.19 seconds!\n", + "\r", + "Evaluation on dev at Epoch 5/10. Step:770/1540: \n", + "\r", + "AccMetric: acc=0.7465596330275229\n", + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=28.0), HTML(value='')), layout=Layout(dis…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Evaluate data in 0.14 seconds!\n", + "\r", + "Evaluation on dev at Epoch 6/10. Step:924/1540: \n", + "\r", + "AccMetric: acc=0.7454128440366973\n", + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=28.0), HTML(value='')), layout=Layout(dis…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Evaluate data in 0.43 seconds!\n", + "\r", + "Evaluation on dev at Epoch 7/10. Step:1078/1540: \n", + "\r", + "AccMetric: acc=0.7488532110091743\n", + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=28.0), HTML(value='')), layout=Layout(dis…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Evaluate data in 0.21 seconds!\n", + "\r", + "Evaluation on dev at Epoch 8/10. Step:1232/1540: \n", + "\r", + "AccMetric: acc=0.7431192660550459\n", + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=28.0), HTML(value='')), layout=Layout(dis…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Evaluate data in 0.1 seconds!\n", + "\r", + "Evaluation on dev at Epoch 9/10. Step:1386/1540: \n", + "\r", + "AccMetric: acc=0.7477064220183486\n", + "\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, layout=Layout(flex='2'), max=28.0), HTML(value='')), layout=Layout(dis…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r", + "Evaluate data in 0.29 seconds!\n", + "\r", + "Evaluation on dev at Epoch 10/10. Step:1540/1540: \n", + "\r", + "AccMetric: acc=0.7465596330275229\n", + "\n", + "\r\n", + "In Epoch:3/Step:462, got best dev performance:\n", + "AccMetric: acc=0.7557339449541285\n", + "Reloaded the best model.\n" + ] + }, + { + "data": { + "text/plain": [ + "{'best_eval': {'AccMetric': {'acc': 0.7557339449541285}},\n", + " 'best_epoch': 3,\n", + " 'best_step': 462,\n", + " 'seconds': 28.68}" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "trainer = Trainer(train_data=train_data, model=model, loss=loss,\n", + " optimizer=optimizer, batch_size=32, dev_data=dev_data,\n", + " metrics=AccMetric(pred=\"pred\", target=\"target\"), device=device)\n", + "trainer.train()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python Now", + "language": "python", + "name": "now" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}