{
"cells": [
{
"cell_type": "markdown",
"id": "fdd7ff16",
"metadata": {},
"source": [
"# T6. fastNLP 与 paddle 或 jittor 的结合\n",
"\n",
" 1 fastNLP 结合 paddle 训练模型\n",
" \n",
" 1.1 关于 paddle 的简单介绍\n",
"\n",
" 1.2 使用 paddle 搭建并训练模型\n",
"\n",
" 2 fastNLP 结合 jittor 训练模型\n",
"\n",
" 2.1 关于 jittor 的简单介绍\n",
"\n",
" 2.2 使用 jittor 搭建并训练模型\n",
"\n",
""
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "08752c5a",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Reusing dataset glue (/remote-home/xrliu/.cache/huggingface/datasets/glue/sst2/1.0.0/dacbe3125aa31d7f70367a07a8a9e72a5a0bfeb5fc42e75c9db75b96da6053ad)\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "6b13d42c39ba455eb370bf2caaa3a264",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/3 [00:00\n",
"\n"
],
"text/plain": [
"\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Processing: 0%| | 0/6000 [00:00 True\n"
]
}
],
"source": [
"import sys\n",
"sys.path.append('..')\n",
"\n",
"from fastNLP import DataSet\n",
"\n",
"dataset = DataSet.from_pandas(sst2data['train'].to_pandas())[:6000]\n",
"\n",
"dataset.apply_more(lambda ins:{'words': ins['sentence'].lower().split(), 'target': ins['label']}, \n",
" progress_bar=\"tqdm\")\n",
"dataset.delete_field('sentence')\n",
"dataset.delete_field('label')\n",
"dataset.delete_field('idx')\n",
"\n",
"from fastNLP import Vocabulary\n",
"\n",
"vocab = Vocabulary()\n",
"vocab.from_dataset(dataset, field_name='words')\n",
"vocab.index_dataset(dataset, field_name='words')\n",
"\n",
"train_dataset, evaluate_dataset = dataset.split(ratio=0.85)\n",
"print(type(train_dataset), isinstance(train_dataset, DataSet))\n",
"\n",
"from fastNLP.io import DataBundle\n",
"\n",
"data_bundle = DataBundle(datasets={'train': train_dataset, 'dev': evaluate_dataset})"
]
},
{
"cell_type": "markdown",
"id": "57a3272f",
"metadata": {},
"source": [
"## 1. fastNLP 结合 paddle 训练模型\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "e31b3198",
"metadata": {},
"outputs": [],
"source": [
"import paddle\n",
"import paddle.nn as nn\n",
"import paddle.nn.functional as F\n",
"\n",
"\n",
"class ClsByPaddle(nn.Layer):\n",
" def __init__(self, vocab_size, embedding_dim, output_dim, hidden_dim=64, dropout=0.5):\n",
" nn.Layer.__init__(self)\n",
" self.hidden_dim = hidden_dim\n",
"\n",
" self.embedding = nn.Embedding(num_embeddings=vocab_size, embedding_dim=embedding_dim)\n",
" \n",
" self.conv1 = nn.Sequential(nn.Conv1D(embedding_dim, 30, 1, padding=0), nn.ReLU())\n",
" self.conv2 = nn.Sequential(nn.Conv1D(embedding_dim, 40, 3, padding=1), nn.ReLU())\n",
" self.conv3 = nn.Sequential(nn.Conv1D(embedding_dim, 50, 5, padding=2), nn.ReLU())\n",
"\n",
" self.mlp = nn.Sequential(('dropout', nn.Dropout(p=dropout)),\n",
" ('linear_1', nn.Linear(120, hidden_dim)),\n",
" ('activate', nn.ReLU()),\n",
" ('linear_2', nn.Linear(hidden_dim, output_dim)))\n",
" \n",
" self.loss_fn = nn.MSELoss()\n",
"\n",
" def forward(self, words):\n",
" output = self.embedding(words).transpose([0, 2, 1])\n",
" conv1, conv2, conv3 = self.conv1(output), self.conv2(output), self.conv3(output)\n",
"\n",
" pool1 = F.max_pool1d(conv1, conv1.shape[-1]).squeeze(2)\n",
" pool2 = F.max_pool1d(conv2, conv2.shape[-1]).squeeze(2)\n",
" pool3 = F.max_pool1d(conv3, conv3.shape[-1]).squeeze(2)\n",
"\n",
" pool = paddle.concat([pool1, pool2, pool3], axis=1)\n",
" output = self.mlp(pool)\n",
" return output\n",
" \n",
" def train_step(self, words, target):\n",
" pred = self(words)\n",
" target = paddle.stack((1 - target, target), axis=1).cast(pred.dtype)\n",
" return {'loss': self.loss_fn(pred, target)}\n",
"\n",
" def evaluate_step(self, words, target):\n",
" pred = self(words)\n",
" pred = paddle.argmax(pred, axis=-1)\n",
" return {'pred': pred, 'target': target}"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "c63b030f",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"W0604 21:02:25.453869 19014 gpu_context.cc:278] Please NOTE: device: 0, GPU Compute Capability: 6.1, Driver API Version: 11.1, Runtime API Version: 10.2\n",
"W0604 21:02:26.061690 19014 gpu_context.cc:306] device: 0, cuDNN Version: 7.6.\n"
]
},
{
"data": {
"text/plain": [
"ClsByPaddle(\n",
" (embedding): Embedding(8458, 100, sparse=False)\n",
" (conv1): Sequential(\n",
" (0): Conv1D(100, 30, kernel_size=[1], data_format=NCL)\n",
" (1): ReLU()\n",
" )\n",
" (conv2): Sequential(\n",
" (0): Conv1D(100, 40, kernel_size=[3], padding=1, data_format=NCL)\n",
" (1): ReLU()\n",
" )\n",
" (conv3): Sequential(\n",
" (0): Conv1D(100, 50, kernel_size=[5], padding=2, data_format=NCL)\n",
" (1): ReLU()\n",
" )\n",
" (mlp): Sequential(\n",
" (dropout): Dropout(p=0.5, axis=None, mode=upscale_in_train)\n",
" (linear_1): Linear(in_features=120, out_features=64, dtype=float32)\n",
" (activate): ReLU()\n",
" (linear_2): Linear(in_features=64, out_features=2, dtype=float32)\n",
" )\n",
" (loss_fn): MSELoss()\n",
")"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model = ClsByPaddle(vocab_size=len(vocab), embedding_dim=100, output_dim=2)\n",
"\n",
"model"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "2997c0aa",
"metadata": {},
"outputs": [],
"source": [
"from paddle.optimizer import AdamW\n",
"\n",
"optimizers = AdamW(parameters=model.parameters(), learning_rate=5e-4)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "ead35fb8",
"metadata": {},
"outputs": [],
"source": [
"from fastNLP import prepare_paddle_dataloader\n",
"\n",
"train_dataloader = prepare_paddle_dataloader(train_dataset, batch_size=16, shuffle=True)\n",
"evaluate_dataloader = prepare_paddle_dataloader(evaluate_dataset, batch_size=16)\n",
"\n",
"# dl_bundle = prepare_paddle_dataloader(data_bundle, batch_size=16, shuffle=True)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "25e8da83",
"metadata": {},
"outputs": [],
"source": [
"from fastNLP import Trainer, Accuracy\n",
"\n",
"trainer = Trainer(\n",
" model=model,\n",
" driver='paddle',\n",
" device='gpu', # 'cpu', 'gpu', 'gpu:x'\n",
" n_epochs=10,\n",
" optimizers=optimizers,\n",
" train_dataloader=train_dataloader, # dl_bundle['train'],\n",
" evaluate_dataloaders=evaluate_dataloader, # dl_bundle['dev'], \n",
" metrics={'acc': Accuracy()}\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "d63c5d74",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"[21:03:08] INFO Running evaluator sanity check for 2 batches. trainer.py:596\n",
"\n"
],
"text/plain": [
"\u001b[2;36m[21:03:08]\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m Running evaluator sanity check for \u001b[1;36m2\u001b[0m batches. \u001b]8;id=894986;file://../fastNLP/core/controllers/trainer.py\u001b\\\u001b[2mtrainer.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=567751;file://../fastNLP/core/controllers/trainer.py#596\u001b\\\u001b[2m596\u001b[0m\u001b]8;;\u001b\\\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Output()"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"/remote-home/xrliu/anaconda3/envs/demo/lib/python3.7/site-packages/ipywidgets/widgets/widget_\n",
"output.py:111: DeprecationWarning: Kernel._parent_header is deprecated in ipykernel 6. Use \n",
".get_parent()\n",
" if ip and hasattr(ip, 'kernel') and hasattr(ip.kernel, '_parent_header'):\n",
"\n"
],
"text/plain": [
"/remote-home/xrliu/anaconda3/envs/demo/lib/python3.7/site-packages/ipywidgets/widgets/widget_\n",
"output.py:111: DeprecationWarning: Kernel._parent_header is deprecated in ipykernel 6. Use \n",
".get_parent()\n",
" if ip and hasattr(ip, 'kernel') and hasattr(ip.kernel, '_parent_header'):\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"/remote-home/xrliu/anaconda3/envs/demo/lib/python3.7/site-packages/ipywidgets/widgets/widget_\n",
"output.py:112: DeprecationWarning: Kernel._parent_header is deprecated in ipykernel 6. Use \n",
".get_parent()\n",
" self.msg_id = ip.kernel._parent_header['header']['msg_id']\n",
"\n"
],
"text/plain": [
"/remote-home/xrliu/anaconda3/envs/demo/lib/python3.7/site-packages/ipywidgets/widgets/widget_\n",
"output.py:112: DeprecationWarning: Kernel._parent_header is deprecated in ipykernel 6. Use \n",
".get_parent()\n",
" self.msg_id = ip.kernel._parent_header['header']['msg_id']\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"/remote-home/xrliu/anaconda3/envs/demo/lib/python3.7/site-packages/paddle/tensor/creation.py:\n",
"125: DeprecationWarning: `np.object` is a deprecated alias for the builtin `object`. To \n",
"silence this warning, use `object` by itself. Doing this will not modify any behavior and is \n",
"safe. \n",
"Deprecated in NumPy 1.20; for more details and guidance: \n",
"https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
" if data.dtype == np.object:\n",
"\n"
],
"text/plain": [
"/remote-home/xrliu/anaconda3/envs/demo/lib/python3.7/site-packages/paddle/tensor/creation.py:\n",
"125: DeprecationWarning: `np.object` is a deprecated alias for the builtin `object`. To \n",
"silence this warning, use `object` by itself. Doing this will not modify any behavior and is \n",
"safe. \n",
"Deprecated in NumPy 1.20; for more details and guidance: \n",
"https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
" if data.dtype == np.object:\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"\n"
],
"text/plain": []
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Output()"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
"\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"----------------------------- Eval. results on Epoch:1, Batch:0 -----------------------------\n",
"\n"
],
"text/plain": [
"----------------------------- Eval. results on Epoch:\u001b[1;36m1\u001b[0m, Batch:\u001b[1;36m0\u001b[0m -----------------------------\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"{\n",
" \"acc#acc\": 0.78125,\n",
" \"total#acc\": 160.0,\n",
" \"correct#acc\": 125.0\n",
"}\n",
"\n"
],
"text/plain": [
"\u001b[1m{\u001b[0m\n",
" \u001b[1;34m\"acc#acc\"\u001b[0m: \u001b[1;36m0.78125\u001b[0m,\n",
" \u001b[1;34m\"total#acc\"\u001b[0m: \u001b[1;36m160.0\u001b[0m,\n",
" \u001b[1;34m\"correct#acc\"\u001b[0m: \u001b[1;36m125.0\u001b[0m\n",
"\u001b[1m}\u001b[0m\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
"\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"----------------------------- Eval. results on Epoch:2, Batch:0 -----------------------------\n",
"\n"
],
"text/plain": [
"----------------------------- Eval. results on Epoch:\u001b[1;36m2\u001b[0m, Batch:\u001b[1;36m0\u001b[0m -----------------------------\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"{\n",
" \"acc#acc\": 0.7875,\n",
" \"total#acc\": 160.0,\n",
" \"correct#acc\": 126.0\n",
"}\n",
"\n"
],
"text/plain": [
"\u001b[1m{\u001b[0m\n",
" \u001b[1;34m\"acc#acc\"\u001b[0m: \u001b[1;36m0.7875\u001b[0m,\n",
" \u001b[1;34m\"total#acc\"\u001b[0m: \u001b[1;36m160.0\u001b[0m,\n",
" \u001b[1;34m\"correct#acc\"\u001b[0m: \u001b[1;36m126.0\u001b[0m\n",
"\u001b[1m}\u001b[0m\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
"\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"----------------------------- Eval. results on Epoch:3, Batch:0 -----------------------------\n",
"\n"
],
"text/plain": [
"----------------------------- Eval. results on Epoch:\u001b[1;36m3\u001b[0m, Batch:\u001b[1;36m0\u001b[0m -----------------------------\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"{\n",
" \"acc#acc\": 0.8,\n",
" \"total#acc\": 160.0,\n",
" \"correct#acc\": 128.0\n",
"}\n",
"\n"
],
"text/plain": [
"\u001b[1m{\u001b[0m\n",
" \u001b[1;34m\"acc#acc\"\u001b[0m: \u001b[1;36m0.8\u001b[0m,\n",
" \u001b[1;34m\"total#acc\"\u001b[0m: \u001b[1;36m160.0\u001b[0m,\n",
" \u001b[1;34m\"correct#acc\"\u001b[0m: \u001b[1;36m128.0\u001b[0m\n",
"\u001b[1m}\u001b[0m\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
"\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"----------------------------- Eval. results on Epoch:4, Batch:0 -----------------------------\n",
"\n"
],
"text/plain": [
"----------------------------- Eval. results on Epoch:\u001b[1;36m4\u001b[0m, Batch:\u001b[1;36m0\u001b[0m -----------------------------\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"{\n",
" \"acc#acc\": 0.79375,\n",
" \"total#acc\": 160.0,\n",
" \"correct#acc\": 127.0\n",
"}\n",
"\n"
],
"text/plain": [
"\u001b[1m{\u001b[0m\n",
" \u001b[1;34m\"acc#acc\"\u001b[0m: \u001b[1;36m0.79375\u001b[0m,\n",
" \u001b[1;34m\"total#acc\"\u001b[0m: \u001b[1;36m160.0\u001b[0m,\n",
" \u001b[1;34m\"correct#acc\"\u001b[0m: \u001b[1;36m127.0\u001b[0m\n",
"\u001b[1m}\u001b[0m\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
"\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"----------------------------- Eval. results on Epoch:5, Batch:0 -----------------------------\n",
"\n"
],
"text/plain": [
"----------------------------- Eval. results on Epoch:\u001b[1;36m5\u001b[0m, Batch:\u001b[1;36m0\u001b[0m -----------------------------\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"{\n",
" \"acc#acc\": 0.81875,\n",
" \"total#acc\": 160.0,\n",
" \"correct#acc\": 131.0\n",
"}\n",
"\n"
],
"text/plain": [
"\u001b[1m{\u001b[0m\n",
" \u001b[1;34m\"acc#acc\"\u001b[0m: \u001b[1;36m0.81875\u001b[0m,\n",
" \u001b[1;34m\"total#acc\"\u001b[0m: \u001b[1;36m160.0\u001b[0m,\n",
" \u001b[1;34m\"correct#acc\"\u001b[0m: \u001b[1;36m131.0\u001b[0m\n",
"\u001b[1m}\u001b[0m\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
"\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"----------------------------- Eval. results on Epoch:6, Batch:0 -----------------------------\n",
"\n"
],
"text/plain": [
"----------------------------- Eval. results on Epoch:\u001b[1;36m6\u001b[0m, Batch:\u001b[1;36m0\u001b[0m -----------------------------\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"{\n",
" \"acc#acc\": 0.8,\n",
" \"total#acc\": 160.0,\n",
" \"correct#acc\": 128.0\n",
"}\n",
"\n"
],
"text/plain": [
"\u001b[1m{\u001b[0m\n",
" \u001b[1;34m\"acc#acc\"\u001b[0m: \u001b[1;36m0.8\u001b[0m,\n",
" \u001b[1;34m\"total#acc\"\u001b[0m: \u001b[1;36m160.0\u001b[0m,\n",
" \u001b[1;34m\"correct#acc\"\u001b[0m: \u001b[1;36m128.0\u001b[0m\n",
"\u001b[1m}\u001b[0m\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
"\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"----------------------------- Eval. results on Epoch:7, Batch:0 -----------------------------\n",
"\n"
],
"text/plain": [
"----------------------------- Eval. results on Epoch:\u001b[1;36m7\u001b[0m, Batch:\u001b[1;36m0\u001b[0m -----------------------------\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"{\n",
" \"acc#acc\": 0.80625,\n",
" \"total#acc\": 160.0,\n",
" \"correct#acc\": 129.0\n",
"}\n",
"\n"
],
"text/plain": [
"\u001b[1m{\u001b[0m\n",
" \u001b[1;34m\"acc#acc\"\u001b[0m: \u001b[1;36m0.80625\u001b[0m,\n",
" \u001b[1;34m\"total#acc\"\u001b[0m: \u001b[1;36m160.0\u001b[0m,\n",
" \u001b[1;34m\"correct#acc\"\u001b[0m: \u001b[1;36m129.0\u001b[0m\n",
"\u001b[1m}\u001b[0m\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
"\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"----------------------------- Eval. results on Epoch:8, Batch:0 -----------------------------\n",
"\n"
],
"text/plain": [
"----------------------------- Eval. results on Epoch:\u001b[1;36m8\u001b[0m, Batch:\u001b[1;36m0\u001b[0m -----------------------------\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"{\n",
" \"acc#acc\": 0.79375,\n",
" \"total#acc\": 160.0,\n",
" \"correct#acc\": 127.0\n",
"}\n",
"\n"
],
"text/plain": [
"\u001b[1m{\u001b[0m\n",
" \u001b[1;34m\"acc#acc\"\u001b[0m: \u001b[1;36m0.79375\u001b[0m,\n",
" \u001b[1;34m\"total#acc\"\u001b[0m: \u001b[1;36m160.0\u001b[0m,\n",
" \u001b[1;34m\"correct#acc\"\u001b[0m: \u001b[1;36m127.0\u001b[0m\n",
"\u001b[1m}\u001b[0m\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
"\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"----------------------------- Eval. results on Epoch:9, Batch:0 -----------------------------\n",
"\n"
],
"text/plain": [
"----------------------------- Eval. results on Epoch:\u001b[1;36m9\u001b[0m, Batch:\u001b[1;36m0\u001b[0m -----------------------------\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"{\n",
" \"acc#acc\": 0.7875,\n",
" \"total#acc\": 160.0,\n",
" \"correct#acc\": 126.0\n",
"}\n",
"\n"
],
"text/plain": [
"\u001b[1m{\u001b[0m\n",
" \u001b[1;34m\"acc#acc\"\u001b[0m: \u001b[1;36m0.7875\u001b[0m,\n",
" \u001b[1;34m\"total#acc\"\u001b[0m: \u001b[1;36m160.0\u001b[0m,\n",
" \u001b[1;34m\"correct#acc\"\u001b[0m: \u001b[1;36m126.0\u001b[0m\n",
"\u001b[1m}\u001b[0m\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
"\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"---------------------------- Eval. results on Epoch:10, Batch:0 -----------------------------\n",
"\n"
],
"text/plain": [
"---------------------------- Eval. results on Epoch:\u001b[1;36m10\u001b[0m, Batch:\u001b[1;36m0\u001b[0m -----------------------------\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"{\n",
" \"acc#acc\": 0.8,\n",
" \"total#acc\": 160.0,\n",
" \"correct#acc\": 128.0\n",
"}\n",
"\n"
],
"text/plain": [
"\u001b[1m{\u001b[0m\n",
" \u001b[1;34m\"acc#acc\"\u001b[0m: \u001b[1;36m0.8\u001b[0m,\n",
" \u001b[1;34m\"total#acc\"\u001b[0m: \u001b[1;36m160.0\u001b[0m,\n",
" \u001b[1;34m\"correct#acc\"\u001b[0m: \u001b[1;36m128.0\u001b[0m\n",
"\u001b[1m}\u001b[0m\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"\n"
],
"text/plain": []
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
"\n"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"trainer.run(num_eval_batch_per_dl=10) "
]
},
{
"cell_type": "markdown",
"id": "cb9a0b3c",
"metadata": {},
"source": [
"## 2. fastNLP 结合 jittor 训练模型"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "c600191d",
"metadata": {},
"outputs": [],
"source": [
"import jittor\n",
"import jittor.nn as nn\n",
"\n",
"from jittor import Module\n",
"\n",
"\n",
"class ClsByJittor(Module):\n",
" def __init__(self, vocab_size, embedding_dim, output_dim, hidden_dim=64, num_layers=2, dropout=0.5):\n",
" Module.__init__(self)\n",
" self.hidden_dim = hidden_dim\n",
"\n",
" self.embedding = nn.Embedding(num=vocab_size, dim=embedding_dim)\n",
" self.lstm = nn.LSTM(input_size=embedding_dim, hidden_size=hidden_dim, batch_first=True, # 默认 batch_first=False\n",
" num_layers=num_layers, bidirectional=True, dropout=dropout)\n",
" self.mlp = nn.Sequential([nn.Dropout(p=dropout),\n",
" nn.Linear(hidden_dim * 2, hidden_dim * 2),\n",
" nn.ReLU(),\n",
" nn.Linear(hidden_dim * 2, output_dim),\n",
" nn.Sigmoid(),])\n",
"\n",
" self.loss_fn = nn.MSELoss()\n",
"\n",
" def execute(self, words):\n",
" output = self.embedding(words)\n",
" output, (hidden, cell) = self.lstm(output)\n",
" output = self.mlp(jittor.concat((hidden[-1], hidden[-2]), dim=1))\n",
" return output\n",
" \n",
" def train_step(self, words, target):\n",
" pred = self(words)\n",
" target = jittor.stack((1 - target, target), dim=1)\n",
" return {'loss': self.loss_fn(pred, target)}\n",
"\n",
" def evaluate_step(self, words, target):\n",
" pred = self(words)\n",
" pred = jittor.argmax(pred, dim=-1)[0]\n",
" return {'pred': pred, 'target': target}"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "a94ed8c4",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"ClsByJittor(\n",
" embedding: Embedding(8458, 100)\n",
" lstm: LSTM(100, 64, 2, bias=True, batch_first=True, dropout=0.5, bidirectional=True, proj_size=0)\n",
" mlp: Sequential(\n",
" 0: Dropout(0.5, is_train=False)\n",
" 1: Linear(128, 128, float32[128,], None)\n",
" 2: relu()\n",
" 3: Linear(128, 2, float32[2,], None)\n",
" 4: Sigmoid()\n",
" )\n",
" loss_fn: MSELoss(mean)\n",
")"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"model = ClsByJittor(vocab_size=len(vocab), embedding_dim=100, output_dim=2)\n",
"\n",
"model"
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "6d15ebc1",
"metadata": {},
"outputs": [],
"source": [
"from jittor.optim import AdamW\n",
"\n",
"optimizers = AdamW(params=model.parameters(), lr=5e-3)"
]
},
{
"cell_type": "code",
"execution_count": 14,
"id": "95d8d09e",
"metadata": {},
"outputs": [],
"source": [
"from fastNLP import prepare_jittor_dataloader\n",
"\n",
"train_dataloader = prepare_jittor_dataloader(train_dataset, batch_size=16, shuffle=True)\n",
"evaluate_dataloader = prepare_jittor_dataloader(evaluate_dataset, batch_size=16)\n",
"\n",
"# dl_bundle = prepare_jittor_dataloader(data_bundle, batch_size=16, shuffle=True)"
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "917eab81",
"metadata": {},
"outputs": [],
"source": [
"from fastNLP import Trainer, Accuracy\n",
"\n",
"trainer = Trainer(\n",
" model=model,\n",
" driver='jittor',\n",
" device='gpu', # 'cpu', 'gpu', 'cuda'\n",
" n_epochs=10,\n",
" optimizers=optimizers,\n",
" train_dataloader=train_dataloader, # dl_bundle['train'],\n",
" evaluate_dataloaders=evaluate_dataloader, # dl_bundle['dev'],\n",
" metrics={'acc': Accuracy()}\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 16,
"id": "f7c4ac5a",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"[21:05:51] INFO Running evaluator sanity check for 2 batches. trainer.py:596\n",
"\n"
],
"text/plain": [
"\u001b[2;36m[21:05:51]\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m Running evaluator sanity check for \u001b[1;36m2\u001b[0m batches. \u001b]8;id=69759;file://../fastNLP/core/controllers/trainer.py\u001b\\\u001b[2mtrainer.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=202322;file://../fastNLP/core/controllers/trainer.py#596\u001b\\\u001b[2m596\u001b[0m\u001b]8;;\u001b\\\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Output()"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"\n"
],
"text/plain": []
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Output()"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n",
"Compiling Operators(5/6) used: 8.31s eta: 1.66s 6/6) used: 9.33s eta: 0s \n",
"\n",
"Compiling Operators(31/31) used: 7.31s eta: 0s \n"
]
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
"\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"----------------------------- Eval. results on Epoch:1, Batch:0 -----------------------------\n",
"\n"
],
"text/plain": [
"----------------------------- Eval. results on Epoch:\u001b[1;36m1\u001b[0m, Batch:\u001b[1;36m0\u001b[0m -----------------------------\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"{\n",
" \"acc#acc\": 0.61875,\n",
" \"total#acc\": 160,\n",
" \"correct#acc\": 99\n",
"}\n",
"\n"
],
"text/plain": [
"\u001b[1m{\u001b[0m\n",
" \u001b[1;34m\"acc#acc\"\u001b[0m: \u001b[1;36m0.61875\u001b[0m,\n",
" \u001b[1;34m\"total#acc\"\u001b[0m: \u001b[1;36m160\u001b[0m,\n",
" \u001b[1;34m\"correct#acc\"\u001b[0m: \u001b[1;36m99\u001b[0m\n",
"\u001b[1m}\u001b[0m\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
"\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"----------------------------- Eval. results on Epoch:2, Batch:0 -----------------------------\n",
"\n"
],
"text/plain": [
"----------------------------- Eval. results on Epoch:\u001b[1;36m2\u001b[0m, Batch:\u001b[1;36m0\u001b[0m -----------------------------\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"{\n",
" \"acc#acc\": 0.7,\n",
" \"total#acc\": 160,\n",
" \"correct#acc\": 112\n",
"}\n",
"\n"
],
"text/plain": [
"\u001b[1m{\u001b[0m\n",
" \u001b[1;34m\"acc#acc\"\u001b[0m: \u001b[1;36m0.7\u001b[0m,\n",
" \u001b[1;34m\"total#acc\"\u001b[0m: \u001b[1;36m160\u001b[0m,\n",
" \u001b[1;34m\"correct#acc\"\u001b[0m: \u001b[1;36m112\u001b[0m\n",
"\u001b[1m}\u001b[0m\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
"\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"----------------------------- Eval. results on Epoch:3, Batch:0 -----------------------------\n",
"\n"
],
"text/plain": [
"----------------------------- Eval. results on Epoch:\u001b[1;36m3\u001b[0m, Batch:\u001b[1;36m0\u001b[0m -----------------------------\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"{\n",
" \"acc#acc\": 0.725,\n",
" \"total#acc\": 160,\n",
" \"correct#acc\": 116\n",
"}\n",
"\n"
],
"text/plain": [
"\u001b[1m{\u001b[0m\n",
" \u001b[1;34m\"acc#acc\"\u001b[0m: \u001b[1;36m0.725\u001b[0m,\n",
" \u001b[1;34m\"total#acc\"\u001b[0m: \u001b[1;36m160\u001b[0m,\n",
" \u001b[1;34m\"correct#acc\"\u001b[0m: \u001b[1;36m116\u001b[0m\n",
"\u001b[1m}\u001b[0m\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
"\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"----------------------------- Eval. results on Epoch:4, Batch:0 -----------------------------\n",
"\n"
],
"text/plain": [
"----------------------------- Eval. results on Epoch:\u001b[1;36m4\u001b[0m, Batch:\u001b[1;36m0\u001b[0m -----------------------------\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"{\n",
" \"acc#acc\": 0.74375,\n",
" \"total#acc\": 160,\n",
" \"correct#acc\": 119\n",
"}\n",
"\n"
],
"text/plain": [
"\u001b[1m{\u001b[0m\n",
" \u001b[1;34m\"acc#acc\"\u001b[0m: \u001b[1;36m0.74375\u001b[0m,\n",
" \u001b[1;34m\"total#acc\"\u001b[0m: \u001b[1;36m160\u001b[0m,\n",
" \u001b[1;34m\"correct#acc\"\u001b[0m: \u001b[1;36m119\u001b[0m\n",
"\u001b[1m}\u001b[0m\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
"\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"----------------------------- Eval. results on Epoch:5, Batch:0 -----------------------------\n",
"\n"
],
"text/plain": [
"----------------------------- Eval. results on Epoch:\u001b[1;36m5\u001b[0m, Batch:\u001b[1;36m0\u001b[0m -----------------------------\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"{\n",
" \"acc#acc\": 0.75625,\n",
" \"total#acc\": 160,\n",
" \"correct#acc\": 121\n",
"}\n",
"\n"
],
"text/plain": [
"\u001b[1m{\u001b[0m\n",
" \u001b[1;34m\"acc#acc\"\u001b[0m: \u001b[1;36m0.75625\u001b[0m,\n",
" \u001b[1;34m\"total#acc\"\u001b[0m: \u001b[1;36m160\u001b[0m,\n",
" \u001b[1;34m\"correct#acc\"\u001b[0m: \u001b[1;36m121\u001b[0m\n",
"\u001b[1m}\u001b[0m\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
"\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"----------------------------- Eval. results on Epoch:6, Batch:0 -----------------------------\n",
"\n"
],
"text/plain": [
"----------------------------- Eval. results on Epoch:\u001b[1;36m6\u001b[0m, Batch:\u001b[1;36m0\u001b[0m -----------------------------\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"{\n",
" \"acc#acc\": 0.75625,\n",
" \"total#acc\": 160,\n",
" \"correct#acc\": 121\n",
"}\n",
"\n"
],
"text/plain": [
"\u001b[1m{\u001b[0m\n",
" \u001b[1;34m\"acc#acc\"\u001b[0m: \u001b[1;36m0.75625\u001b[0m,\n",
" \u001b[1;34m\"total#acc\"\u001b[0m: \u001b[1;36m160\u001b[0m,\n",
" \u001b[1;34m\"correct#acc\"\u001b[0m: \u001b[1;36m121\u001b[0m\n",
"\u001b[1m}\u001b[0m\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
"\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"----------------------------- Eval. results on Epoch:7, Batch:0 -----------------------------\n",
"\n"
],
"text/plain": [
"----------------------------- Eval. results on Epoch:\u001b[1;36m7\u001b[0m, Batch:\u001b[1;36m0\u001b[0m -----------------------------\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"{\n",
" \"acc#acc\": 0.73125,\n",
" \"total#acc\": 160,\n",
" \"correct#acc\": 117\n",
"}\n",
"\n"
],
"text/plain": [
"\u001b[1m{\u001b[0m\n",
" \u001b[1;34m\"acc#acc\"\u001b[0m: \u001b[1;36m0.73125\u001b[0m,\n",
" \u001b[1;34m\"total#acc\"\u001b[0m: \u001b[1;36m160\u001b[0m,\n",
" \u001b[1;34m\"correct#acc\"\u001b[0m: \u001b[1;36m117\u001b[0m\n",
"\u001b[1m}\u001b[0m\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
"\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"----------------------------- Eval. results on Epoch:8, Batch:0 -----------------------------\n",
"\n"
],
"text/plain": [
"----------------------------- Eval. results on Epoch:\u001b[1;36m8\u001b[0m, Batch:\u001b[1;36m0\u001b[0m -----------------------------\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"{\n",
" \"acc#acc\": 0.7625,\n",
" \"total#acc\": 160,\n",
" \"correct#acc\": 122\n",
"}\n",
"\n"
],
"text/plain": [
"\u001b[1m{\u001b[0m\n",
" \u001b[1;34m\"acc#acc\"\u001b[0m: \u001b[1;36m0.7625\u001b[0m,\n",
" \u001b[1;34m\"total#acc\"\u001b[0m: \u001b[1;36m160\u001b[0m,\n",
" \u001b[1;34m\"correct#acc\"\u001b[0m: \u001b[1;36m122\u001b[0m\n",
"\u001b[1m}\u001b[0m\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
"\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"----------------------------- Eval. results on Epoch:9, Batch:0 -----------------------------\n",
"\n"
],
"text/plain": [
"----------------------------- Eval. results on Epoch:\u001b[1;36m9\u001b[0m, Batch:\u001b[1;36m0\u001b[0m -----------------------------\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"{\n",
" \"acc#acc\": 0.74375,\n",
" \"total#acc\": 160,\n",
" \"correct#acc\": 119\n",
"}\n",
"\n"
],
"text/plain": [
"\u001b[1m{\u001b[0m\n",
" \u001b[1;34m\"acc#acc\"\u001b[0m: \u001b[1;36m0.74375\u001b[0m,\n",
" \u001b[1;34m\"total#acc\"\u001b[0m: \u001b[1;36m160\u001b[0m,\n",
" \u001b[1;34m\"correct#acc\"\u001b[0m: \u001b[1;36m119\u001b[0m\n",
"\u001b[1m}\u001b[0m\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
"\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"---------------------------- Eval. results on Epoch:10, Batch:0 -----------------------------\n",
"\n"
],
"text/plain": [
"---------------------------- Eval. results on Epoch:\u001b[1;36m10\u001b[0m, Batch:\u001b[1;36m0\u001b[0m -----------------------------\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"{\n",
" \"acc#acc\": 0.7625,\n",
" \"total#acc\": 160,\n",
" \"correct#acc\": 122\n",
"}\n",
"\n"
],
"text/plain": [
"\u001b[1m{\u001b[0m\n",
" \u001b[1;34m\"acc#acc\"\u001b[0m: \u001b[1;36m0.7625\u001b[0m,\n",
" \u001b[1;34m\"total#acc\"\u001b[0m: \u001b[1;36m160\u001b[0m,\n",
" \u001b[1;34m\"correct#acc\"\u001b[0m: \u001b[1;36m122\u001b[0m\n",
"\u001b[1m}\u001b[0m\n"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"\n"
],
"text/plain": []
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"\n",
"\n"
],
"text/plain": [
"\n"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"trainer.run(num_eval_batch_per_dl=10)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "3df5f425",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.13"
}
},
"nbformat": 4,
"nbformat_minor": 5
}