@@ -56,7 +56,7 @@ def is_paddle_dtype_str(dtype): | |||||
def _get_dtype(ele_dtype, dtype, class_name): | def _get_dtype(ele_dtype, dtype, class_name): | ||||
if not (ele_dtype is not None or is_number_or_numpy_number(ele_dtype) or is_paddle_tensor(ele_dtype) or is_paddle_dtype_str(ele_dtype)): | |||||
if not (ele_dtype is None or is_number_or_numpy_number(ele_dtype) or is_paddle_tensor(ele_dtype) or is_paddle_dtype_str(ele_dtype)): | |||||
raise EleDtypeUnsupportedError(f"`{class_name}` only supports padding python numbers " | raise EleDtypeUnsupportedError(f"`{class_name}` only supports padding python numbers " | ||||
f"or numpy numbers or paddle.Tensor but get `{ele_dtype}`.") | f"or numpy numbers or paddle.Tensor but get `{ele_dtype}`.") | ||||
@@ -8,11 +8,12 @@ from typing import Callable, List, Optional, Union, Dict, Sequence | |||||
from fastNLP.envs.imports import _NEED_IMPORT_PADDLE | from fastNLP.envs.imports import _NEED_IMPORT_PADDLE | ||||
if _NEED_IMPORT_PADDLE: | if _NEED_IMPORT_PADDLE: | ||||
from paddle.io import DataLoader, Dataset | |||||
from paddle.io import DataLoader, Dataset, Sampler | |||||
from paddle.fluid.dataloader.collate import default_collate_fn | from paddle.fluid.dataloader.collate import default_collate_fn | ||||
else: | else: | ||||
from fastNLP.core.utils.dummy_class import DummyClass as Dataset | from fastNLP.core.utils.dummy_class import DummyClass as Dataset | ||||
from fastNLP.core.utils.dummy_class import DummyClass as DataLoader | from fastNLP.core.utils.dummy_class import DummyClass as DataLoader | ||||
from fastNLP.core.utils.dummy_class import DummyClass as Sampler | |||||
from fastNLP.core.collators.collator import Collator | from fastNLP.core.collators.collator import Collator | ||||
from fastNLP.core.dataloaders.utils import indice_collate_wrapper | from fastNLP.core.dataloaders.utils import indice_collate_wrapper | ||||
@@ -58,6 +59,9 @@ class PaddleDataLoader(DataLoader): | |||||
if batch_sampler is None: | if batch_sampler is None: | ||||
batch_sampler = RandomBatchSampler(dataset, batch_size=batch_size, shuffle=shuffle, | batch_sampler = RandomBatchSampler(dataset, batch_size=batch_size, shuffle=shuffle, | ||||
drop_last=drop_last) | drop_last=drop_last) | ||||
batch_size = 1 | |||||
shuffle = False | |||||
drop_last = False | |||||
super(PaddleDataLoader, self).__init__(dataset=dataset, feed_list=feed_list, places=places, | super(PaddleDataLoader, self).__init__(dataset=dataset, feed_list=feed_list, places=places, | ||||
return_list=return_list, batch_sampler=batch_sampler, | return_list=return_list, batch_sampler=batch_sampler, | ||||
@@ -1,12 +1,12 @@ | |||||
import os | import os | ||||
import shutil | |||||
from typing import List, Union, Optional, Dict, Tuple, Callable | from typing import List, Union, Optional, Dict, Tuple, Callable | ||||
from fastNLP.core.utils.paddle_utils import get_device_from_visible | |||||
from .paddle_driver import PaddleDriver | from .paddle_driver import PaddleDriver | ||||
from .fleet_launcher import FleetLauncher | from .fleet_launcher import FleetLauncher | ||||
from .utils import ( | from .utils import ( | ||||
_FleetWrappingModel, | _FleetWrappingModel, | ||||
get_device_from_visible, | |||||
reset_seed, | reset_seed, | ||||
replace_sampler, | replace_sampler, | ||||
replace_batch_sampler, | replace_batch_sampler, | ||||
@@ -17,8 +17,8 @@ from fastNLP.envs.imports import _NEED_IMPORT_PADDLE | |||||
from fastNLP.core.utils import ( | from fastNLP.core.utils import ( | ||||
auto_param_call, | auto_param_call, | ||||
check_user_specific_params, | check_user_specific_params, | ||||
paddle_move_data_to_device, | |||||
is_in_paddle_dist | |||||
is_in_paddle_dist, | |||||
is_in_paddle_dist, | |||||
) | ) | ||||
from fastNLP.envs.distributed import rank_zero_rm | from fastNLP.envs.distributed import rank_zero_rm | ||||
from fastNLP.core.samplers import ( | from fastNLP.core.samplers import ( | ||||
@@ -609,12 +609,6 @@ class PaddleFleetDriver(PaddleDriver): | |||||
def is_distributed(self): | def is_distributed(self): | ||||
return True | return True | ||||
def move_data_to_device(self, batch: 'paddle.Tensor'): | |||||
device = self.data_device | |||||
# 因为设置了CUDA_VISIBLE_DEVICES,可能会引起错误 | |||||
device = get_device_from_visible(device) | |||||
return paddle_move_data_to_device(batch, device) | |||||
@staticmethod | @staticmethod | ||||
def _check_optimizer_legality(optimizers): | def _check_optimizer_legality(optimizers): | ||||
# paddle 存在设置分布式 optimizers 的函数,返回值为 fleet.meta_optimizers.HybridParallelOptimizer | # paddle 存在设置分布式 optimizers 的函数,返回值为 fleet.meta_optimizers.HybridParallelOptimizer | ||||
@@ -637,9 +631,8 @@ class PaddleFleetDriver(PaddleDriver): | |||||
:return: 如果当前不是分布式 driver 直接返回输入的 obj 。如果当前 rank 是接收端(其 global rank 包含在了 dst 中),则返回 | :return: 如果当前不是分布式 driver 直接返回输入的 obj 。如果当前 rank 是接收端(其 global rank 包含在了 dst 中),则返回 | ||||
接收到的参数;如果是 source 端则返回发射的内容;既不是发送端、又不是接收端,则返回 None 。 | 接收到的参数;如果是 source 端则返回发射的内容;既不是发送端、又不是接收端,则返回 None 。 | ||||
""" | """ | ||||
device = self.data_device | |||||
# 因为设置了CUDA_VISIBLE_DEVICES,可能会引起错误 | # 因为设置了CUDA_VISIBLE_DEVICES,可能会引起错误 | ||||
device = get_device_from_visible(device) | |||||
device = get_device_from_visible(self.data_device) | |||||
return fastnlp_paddle_broadcast_object(obj, src, device=device, group=group) | return fastnlp_paddle_broadcast_object(obj, src, device=device, group=group) | ||||
def all_gather(self, obj, group=None) -> List: | def all_gather(self, obj, group=None) -> List: | ||||
@@ -10,7 +10,6 @@ from fastNLP.envs.env import ( | |||||
FASTNLP_DISTRIBUTED_CHECK, | FASTNLP_DISTRIBUTED_CHECK, | ||||
FASTNLP_LOG_LEVEL, | FASTNLP_LOG_LEVEL, | ||||
FASTNLP_GLOBAL_SEED, | FASTNLP_GLOBAL_SEED, | ||||
USER_CUDA_VISIBLE_DEVICES, | |||||
) | ) | ||||
from .utils import ( | from .utils import ( | ||||
find_free_ports, | find_free_ports, | ||||
@@ -42,7 +42,8 @@ def initialize_paddle_driver(driver: str, device: Optional[Union[str, int, List[ | |||||
user_visible_devices = os.getenv("USER_CUDA_VISIBLE_DEVICES") | user_visible_devices = os.getenv("USER_CUDA_VISIBLE_DEVICES") | ||||
if user_visible_devices is None: | if user_visible_devices is None: | ||||
raise RuntimeError("This situation cannot happen, please report a bug to us.") | |||||
raise RuntimeError("`USER_CUDA_VISIBLE_DEVICES` cannot be None, please check if you have set " | |||||
"`FASTNLP_BACKEND` to 'paddle' before using FastNLP.") | |||||
_could_use_device_num = len(user_visible_devices.split(",")) | _could_use_device_num = len(user_visible_devices.split(",")) | ||||
if isinstance(device, int): | if isinstance(device, int): | ||||
if device < 0 and device != -1: | if device < 0 and device != -1: | ||||
@@ -10,7 +10,7 @@ import numpy as np | |||||
from .utils import _build_fp16_env, optimizer_state_to_device, DummyGradScaler | from .utils import _build_fp16_env, optimizer_state_to_device, DummyGradScaler | ||||
from fastNLP.envs.imports import _NEED_IMPORT_PADDLE | from fastNLP.envs.imports import _NEED_IMPORT_PADDLE | ||||
from fastNLP.core.drivers.driver import Driver | from fastNLP.core.drivers.driver import Driver | ||||
from fastNLP.core.utils import apply_to_collection, paddle_move_data_to_device | |||||
from fastNLP.core.utils import apply_to_collection, paddle_move_data_to_device, get_device_from_visible | |||||
from fastNLP.envs import ( | from fastNLP.envs import ( | ||||
FASTNLP_SEED_WORKERS, | FASTNLP_SEED_WORKERS, | ||||
FASTNLP_MODEL_FILENAME, | FASTNLP_MODEL_FILENAME, | ||||
@@ -394,7 +394,8 @@ class PaddleDriver(Driver): | |||||
:return: 将移动到指定机器上的 batch 对象返回; | :return: 将移动到指定机器上的 batch 对象返回; | ||||
""" | """ | ||||
return paddle_move_data_to_device(batch, self.data_device) | |||||
device = get_device_from_visible(self.data_device) | |||||
return paddle_move_data_to_device(batch, device) | |||||
@staticmethod | @staticmethod | ||||
def worker_init_function(worker_id: int, rank: Optional[int] = None) -> None: # pragma: no cover | def worker_init_function(worker_id: int, rank: Optional[int] = None) -> None: # pragma: no cover | ||||
@@ -2,14 +2,14 @@ import os | |||||
from typing import Optional, Dict, Union, Callable, Tuple | from typing import Optional, Dict, Union, Callable, Tuple | ||||
from .paddle_driver import PaddleDriver | from .paddle_driver import PaddleDriver | ||||
from .utils import replace_batch_sampler, replace_sampler, get_device_from_visible | |||||
from .utils import replace_batch_sampler, replace_sampler | |||||
from fastNLP.envs.imports import _NEED_IMPORT_PADDLE | from fastNLP.envs.imports import _NEED_IMPORT_PADDLE | ||||
from fastNLP.envs.env import USER_CUDA_VISIBLE_DEVICES | from fastNLP.envs.env import USER_CUDA_VISIBLE_DEVICES | ||||
from fastNLP.core.utils import ( | from fastNLP.core.utils import ( | ||||
auto_param_call, | auto_param_call, | ||||
get_device_from_visible, | |||||
get_paddle_gpu_str, | get_paddle_gpu_str, | ||||
get_paddle_device_id, | get_paddle_device_id, | ||||
paddle_move_data_to_device, | |||||
) | ) | ||||
from fastNLP.core.utils.utils import _get_fun_msg | from fastNLP.core.utils.utils import _get_fun_msg | ||||
from fastNLP.core.samplers import ( | from fastNLP.core.samplers import ( | ||||
@@ -39,6 +39,9 @@ class PaddleSingleDriver(PaddleDriver): | |||||
raise ValueError("`paddle.DataParallel` is not supported in `PaddleSingleDriver`") | raise ValueError("`paddle.DataParallel` is not supported in `PaddleSingleDriver`") | ||||
cuda_visible_devices = os.environ.get(USER_CUDA_VISIBLE_DEVICES, None) | cuda_visible_devices = os.environ.get(USER_CUDA_VISIBLE_DEVICES, None) | ||||
if cuda_visible_devices is None: | |||||
raise RuntimeError("`USER_CUDA_VISIBLE_DEVICES` cannot be None, please check if you have set " | |||||
"`FASTNLP_BACKEND` to 'paddle' before using FastNLP.") | |||||
if cuda_visible_devices == "": | if cuda_visible_devices == "": | ||||
device = "cpu" | device = "cpu" | ||||
logger.info("You have set `CUDA_VISIBLE_DEVICES` to '' in system environment variable, and we are gonna to" | logger.info("You have set `CUDA_VISIBLE_DEVICES` to '' in system environment variable, and we are gonna to" | ||||
@@ -54,7 +57,7 @@ class PaddleSingleDriver(PaddleDriver): | |||||
device_id = device | device_id = device | ||||
else: | else: | ||||
device_id = get_paddle_device_id(device) | device_id = get_paddle_device_id(device) | ||||
os.environ["CUDA_VISIBLE_DEVICES"] = os.environ[USER_CUDA_VISIBLE_DEVICES].split(",")[device_id] | |||||
os.environ["CUDA_VISIBLE_DEVICES"] = cuda_visible_devices.split(",")[device_id] | |||||
self.model_device = get_paddle_gpu_str(device) | self.model_device = get_paddle_gpu_str(device) | ||||
self.local_rank = 0 | self.local_rank = 0 | ||||
@@ -65,8 +68,7 @@ class PaddleSingleDriver(PaddleDriver): | |||||
r""" | r""" | ||||
该函数用来初始化训练环境,用于设置当前训练的设备,并将模型迁移到对应设备上。 | 该函数用来初始化训练环境,用于设置当前训练的设备,并将模型迁移到对应设备上。 | ||||
""" | """ | ||||
device = self.model_device | |||||
device = get_device_from_visible(device, output_type=str) | |||||
device = get_device_from_visible(self.model_device, output_type=str) | |||||
paddle.device.set_device(device) | paddle.device.set_device(device) | ||||
self.model.to(device) | self.model.to(device) | ||||
@@ -121,16 +123,6 @@ class PaddleSingleDriver(PaddleDriver): | |||||
else: | else: | ||||
raise RuntimeError(f"There is no `{fn}` method in your {type(self.model)}.") | raise RuntimeError(f"There is no `{fn}` method in your {type(self.model)}.") | ||||
def move_data_to_device(self, batch: 'paddle.Tensor'): | |||||
r""" | |||||
将数据迁移到指定的机器上;batch 可能是 list 也可能 dict ,或其嵌套结构。 | |||||
在 Paddle 中使用可能会引起因与设置的设备不一致而产生的问题,请注意。 | |||||
:return: 将移动到指定机器上的 batch 对象返回; | |||||
""" | |||||
device = get_device_from_visible(self.data_device) | |||||
return paddle_move_data_to_device(batch, device) | |||||
def set_dist_repro_dataloader(self, dataloader, dist: Union[str, ReproducibleBatchSampler, ReproducibleSampler]=None, | def set_dist_repro_dataloader(self, dataloader, dist: Union[str, ReproducibleBatchSampler, ReproducibleSampler]=None, | ||||
reproducible: bool = False): | reproducible: bool = False): | ||||
r""" | r""" | ||||
@@ -6,12 +6,11 @@ import inspect | |||||
import numpy as np | import numpy as np | ||||
from copy import deepcopy | from copy import deepcopy | ||||
from contextlib import ExitStack, closing | from contextlib import ExitStack, closing | ||||
from enum import IntEnum | |||||
from typing import Dict, Optional, Union | |||||
from typing import Dict, Optional | |||||
from fastNLP.envs.imports import _NEED_IMPORT_PADDLE | from fastNLP.envs.imports import _NEED_IMPORT_PADDLE | ||||
from fastNLP.core.utils import get_paddle_device_id, auto_param_call, paddle_to | |||||
from fastNLP.envs.env import FASTNLP_GLOBAL_SEED, FASTNLP_SEED_WORKERS, USER_CUDA_VISIBLE_DEVICES | |||||
from fastNLP.core.utils import auto_param_call, paddle_to | |||||
from fastNLP.envs.env import FASTNLP_GLOBAL_SEED, FASTNLP_SEED_WORKERS | |||||
from fastNLP.core.log import logger | from fastNLP.core.log import logger | ||||
@@ -173,40 +172,6 @@ def find_free_ports(num): | |||||
return None | return None | ||||
def get_device_from_visible(device: Union[str, int], output_type=int): | |||||
""" | |||||
在有 CUDA_VISIBLE_DEVICES 的情况下,获取对应的设备。 | |||||
如 CUDA_VISIBLE_DEVICES=2,3 ,device=3 ,则返回1。 | |||||
:param device: 未转化的设备名 | |||||
:param output_type: 返回值的类型 | |||||
:return: 转化后的设备id | |||||
""" | |||||
if output_type not in [int, str]: | |||||
raise ValueError("Parameter `output_type` should be one of these types: [int, str]") | |||||
if device == "cpu": | |||||
return device | |||||
cuda_visible_devices = os.getenv("CUDA_VISIBLE_DEVICES") | |||||
idx = get_paddle_device_id(device) | |||||
if cuda_visible_devices is None or cuda_visible_devices == "": | |||||
# 这个判断一般不会发生,因为 fastnlp 会为 paddle 强行注入 CUDA_VISIBLE_DEVICES | |||||
raise RuntimeError("This situation should not happen, please report us this bug.") | |||||
else: | |||||
# 利用 USER_CUDA_VISIBLDE_DEVICES 获取用户期望的设备 | |||||
user_visible_devices = os.getenv(USER_CUDA_VISIBLE_DEVICES) | |||||
if user_visible_devices is None: | |||||
raise RuntimeError("This situation cannot happen, please report a bug to us.") | |||||
idx = user_visible_devices.split(",")[idx] | |||||
cuda_visible_devices_list = cuda_visible_devices.split(',') | |||||
if idx not in cuda_visible_devices_list: | |||||
raise ValueError(f"Can't find your devices {idx} in CUDA_VISIBLE_DEVICES[{cuda_visible_devices}].") | |||||
res = cuda_visible_devices_list.index(idx) | |||||
if output_type == int: | |||||
return res | |||||
else: | |||||
return f"gpu:{res}" | |||||
def replace_batch_sampler(dataloader: "DataLoader", batch_sampler: "BatchSampler"): | def replace_batch_sampler(dataloader: "DataLoader", batch_sampler: "BatchSampler"): | ||||
""" | """ | ||||
利用 `batch_sampler` 重新构建一个 DataLoader,起到替换 `batch_sampler` 又不影响原 `dataloader` 的作用。 | 利用 `batch_sampler` 重新构建一个 DataLoader,起到替换 `batch_sampler` 又不影响原 `dataloader` 的作用。 | ||||
@@ -1,11 +1,10 @@ | |||||
from typing import List, Optional, Any | |||||
from typing import List, Any | |||||
import numpy as np | import numpy as np | ||||
from fastNLP.core.metrics.backend import Backend | from fastNLP.core.metrics.backend import Backend | ||||
from fastNLP.core.utils.paddle_utils import paddle_to | |||||
from fastNLP.core.utils.paddle_utils import paddle_to, get_device_from_visible | |||||
from fastNLP.core.metrics.utils import AggregateMethodError | from fastNLP.core.metrics.utils import AggregateMethodError | ||||
from fastNLP.core.drivers.paddle_driver.utils import get_device_from_visible | |||||
from fastNLP.core.drivers.paddle_driver.dist_utils import fastnlp_paddle_all_gather | from fastNLP.core.drivers.paddle_driver.dist_utils import fastnlp_paddle_all_gather | ||||
from fastNLP.envs.imports import _NEED_IMPORT_PADDLE | from fastNLP.envs.imports import _NEED_IMPORT_PADDLE | ||||
@@ -80,7 +79,6 @@ class PaddleBackend(Backend): | |||||
raise ValueError(f"tensor: {tensor} can not convert to ndarray!") | raise ValueError(f"tensor: {tensor} can not convert to ndarray!") | ||||
def move_tensor_to_device(self, tensor, device): | def move_tensor_to_device(self, tensor, device): | ||||
# TODO 如果在这里处理的话,会不会在别的地方引起bug? | |||||
device = get_device_from_visible(device) | device = get_device_from_visible(device) | ||||
return paddle_to(tensor, device) | return paddle_to(tensor, device) | ||||
@@ -2,6 +2,7 @@ __all__ = [ | |||||
'cache_results', | 'cache_results', | ||||
'is_jittor_dataset', | 'is_jittor_dataset', | ||||
'jittor_collate_wraps', | 'jittor_collate_wraps', | ||||
'get_device_from_visible', | |||||
'paddle_to', | 'paddle_to', | ||||
'paddle_move_data_to_device', | 'paddle_move_data_to_device', | ||||
'get_paddle_device_id', | 'get_paddle_device_id', | ||||
@@ -27,7 +28,7 @@ __all__ = [ | |||||
from .cache_results import cache_results | from .cache_results import cache_results | ||||
from .jittor_utils import is_jittor_dataset, jittor_collate_wraps | from .jittor_utils import is_jittor_dataset, jittor_collate_wraps | ||||
from .paddle_utils import paddle_to, paddle_move_data_to_device, get_paddle_device_id, get_paddle_gpu_str, is_in_paddle_dist, \ | |||||
from .paddle_utils import get_device_from_visible, paddle_to, paddle_move_data_to_device, get_paddle_device_id, get_paddle_gpu_str, is_in_paddle_dist, \ | |||||
is_in_fnlp_paddle_dist, is_in_paddle_launch_dist | is_in_fnlp_paddle_dist, is_in_paddle_launch_dist | ||||
from .rich_progress import f_rich_progress | from .rich_progress import f_rich_progress | ||||
from .torch_paddle_utils import torch_paddle_move_data_to_device | from .torch_paddle_utils import torch_paddle_move_data_to_device | ||||
@@ -1,4 +1,5 @@ | |||||
__all__ = [ | __all__ = [ | ||||
"get_device_from_visible", | |||||
"paddle_to", | "paddle_to", | ||||
"paddle_move_data_to_device", | "paddle_move_data_to_device", | ||||
"get_paddle_gpu_str", | "get_paddle_gpu_str", | ||||
@@ -13,13 +14,45 @@ import re | |||||
from typing import Any, Optional, Union | from typing import Any, Optional, Union | ||||
from fastNLP.envs.imports import _NEED_IMPORT_PADDLE | from fastNLP.envs.imports import _NEED_IMPORT_PADDLE | ||||
from fastNLP.envs import FASTNLP_DISTRIBUTED_CHECK, FASTNLP_BACKEND_LAUNCH | |||||
from fastNLP.envs import FASTNLP_DISTRIBUTED_CHECK, FASTNLP_BACKEND_LAUNCH, USER_CUDA_VISIBLE_DEVICES | |||||
if _NEED_IMPORT_PADDLE: | if _NEED_IMPORT_PADDLE: | ||||
import paddle | import paddle | ||||
from .utils import apply_to_collection | from .utils import apply_to_collection | ||||
def get_device_from_visible(device: Union[str, int], output_type=int): | |||||
""" | |||||
在有 CUDA_VISIBLE_DEVICES 的情况下,获取对应的设备。 | |||||
如 CUDA_VISIBLE_DEVICES=2,3 ,device=3 ,则返回1。 | |||||
:param device: 未转化的设备名 | |||||
:param output_type: 返回值的类型 | |||||
:return: 转化后的设备id | |||||
""" | |||||
if output_type not in [int, str]: | |||||
raise ValueError("Parameter `output_type` should be one of these types: [int, str]") | |||||
if device == "cpu": | |||||
return device | |||||
cuda_visible_devices = os.getenv("CUDA_VISIBLE_DEVICES") | |||||
user_visible_devices = os.getenv(USER_CUDA_VISIBLE_DEVICES) | |||||
if user_visible_devices is None: | |||||
raise RuntimeError("`USER_CUDA_VISIBLE_DEVICES` cannot be None, please check if you have set " | |||||
"`FASTNLP_BACKEND` to 'paddle' before using FastNLP.") | |||||
idx = get_paddle_device_id(device) | |||||
# 利用 USER_CUDA_VISIBLDE_DEVICES 获取用户期望的设备 | |||||
if user_visible_devices is None: | |||||
raise RuntimeError("This situation cannot happen, please report a bug to us.") | |||||
idx = user_visible_devices.split(",")[idx] | |||||
cuda_visible_devices_list = cuda_visible_devices.split(',') | |||||
if idx not in cuda_visible_devices_list: | |||||
raise ValueError(f"Can't find your devices {idx} in CUDA_VISIBLE_DEVICES[{cuda_visible_devices}]. ") | |||||
res = cuda_visible_devices_list.index(idx) | |||||
if output_type == int: | |||||
return res | |||||
else: | |||||
return f"gpu:{res}" | |||||
def paddle_to(data, device: Union[str, int]): | def paddle_to(data, device: Union[str, int]): | ||||
""" | """ | ||||
@@ -33,6 +66,7 @@ def paddle_to(data, device: Union[str, int]): | |||||
if device == "cpu": | if device == "cpu": | ||||
return data.cpu() | return data.cpu() | ||||
else: | else: | ||||
# device = get_device_from_visible(device, output_type=int) | |||||
return data.cuda(get_paddle_device_id(device)) | return data.cuda(get_paddle_device_id(device)) | ||||
@@ -14,10 +14,10 @@ def test_get_element_shape_dtype(): | |||||
catalog = _get_element_shape_dtype([np.zeros(3), np.zeros((2, 1))]) | catalog = _get_element_shape_dtype([np.zeros(3), np.zeros((2, 1))]) | ||||
@pytest.mark.parametrize('backend', ['raw', None, 'numpy', 'torch', 'jittor', 'paddle']) | |||||
# @pytest.mark.parametrize('backend', ['raw', None, 'numpy', 'torch', 'jittor', 'paddle']) | |||||
@pytest.mark.parametrize('backend', ['raw', None, 'numpy', 'torch', 'paddle']) | |||||
@pytest.mark.torch | @pytest.mark.torch | ||||
@pytest.mark.paddle | @pytest.mark.paddle | ||||
@pytest.mark.jittor | |||||
def test_get_padder_run(backend): | def test_get_padder_run(backend): | ||||
if not _NEED_IMPORT_TORCH and backend == 'torch': | if not _NEED_IMPORT_TORCH and backend == 'torch': | ||||
pytest.skip("No torch") | pytest.skip("No torch") | ||||
@@ -1,7 +1,7 @@ | |||||
""" | """ | ||||
这个文件测试用户以python -m paddle.distributed.launch 启动的情况 | 这个文件测试用户以python -m paddle.distributed.launch 启动的情况 | ||||
看看有没有用pytest执行的机会 | 看看有没有用pytest执行的机会 | ||||
python -m paddle.distributed.launch --gpus=0,2,3 test_trainer_fleet.py | |||||
FASTNLP_BACKEND=paddle python -m paddle.distributed.launch --gpus=0,2,3 _test_trainer_fleet.py | |||||
""" | """ | ||||
import os | import os | ||||
import sys | import sys | ||||
@@ -1,7 +1,7 @@ | |||||
""" | """ | ||||
这个文件测试用户以python -m paddle.distributed.launch 启动的情况 | 这个文件测试用户以python -m paddle.distributed.launch 启动的情况 | ||||
并且自己初始化了 fleet | 并且自己初始化了 fleet | ||||
python -m paddle.distributed.launch --gpus=0,2,3 test_trainer_fleet_outside.py | |||||
FASTNLP_BACKEND=paddle python -m paddle.distributed.launch --gpus=0,2,3 _test_trainer_fleet_outside.py | |||||
""" | """ | ||||
import os | import os | ||||
import sys | import sys | ||||
@@ -93,5 +93,5 @@ if __name__ == "__main__": | |||||
driver=driver, | driver=driver, | ||||
device=device, | device=device, | ||||
callbacks=callbacks, | callbacks=callbacks, | ||||
n_epochs=30, | |||||
n_epochs=5, | |||||
) | ) |
@@ -27,7 +27,7 @@ class TrainPaddleConfig: | |||||
@pytest.mark.parametrize("driver,device", [("paddle", "cpu"), ("paddle", 1), ("fleet", [0, 1])]) | @pytest.mark.parametrize("driver,device", [("paddle", "cpu"), ("paddle", 1), ("fleet", [0, 1])]) | ||||
# @pytest.mark.parametrize("driver,device", [("fleet", [0, 1])]) | # @pytest.mark.parametrize("driver,device", [("fleet", [0, 1])]) | ||||
@pytest.mark.parametrize("callbacks", [[RichCallback(5)]]) | @pytest.mark.parametrize("callbacks", [[RichCallback(5)]]) | ||||
@pytest.mark.paddle | |||||
@pytest.mark.paddledist | |||||
@magic_argv_env_context | @magic_argv_env_context | ||||
def test_trainer_paddle( | def test_trainer_paddle( | ||||
driver, | driver, | ||||
@@ -58,11 +58,3 @@ class TestPaddle: | |||||
for batch in fdl1: | for batch in fdl1: | ||||
assert batch['image'].shape == [4, 10, 5] | assert batch['image'].shape == [4, 10, 5] | ||||
print(batch) | print(batch) | ||||
def test_v2(self): | |||||
from fastNLP.core.collators import Collator | |||||
logger.setLevel("DEBUG") | |||||
data = [paddle.Tensor(np.random.random((10, 5)).astype('float32')), paddle.Tensor(np.random.random((10, 5)).astype('float32'))] | |||||
col = Collator(backend="jittor") | |||||
res = col(data) | |||||
print(res) |
@@ -370,29 +370,11 @@ class TestDataSetMethods: | |||||
assert os.path.exists("1.csv") == True | assert os.path.exists("1.csv") == True | ||||
os.remove("1.csv") | os.remove("1.csv") | ||||
def test_add_collate_fn(self): | |||||
ds = DataSet({'x': [1, 2, 3], 'y': [4, 5, 6]}) | |||||
def collate_fn(item): | |||||
return item | |||||
ds.add_collate_fn(collate_fn) | |||||
def test_get_collator(self): | |||||
from typing import Callable | |||||
ds = DataSet({'x': [1, 2, 3], 'y': [4, 5, 6]}) | |||||
collate_fn = ds.get_collator() | |||||
assert isinstance(collate_fn, Callable) == True | |||||
def test_add_seq_len(self): | def test_add_seq_len(self): | ||||
ds = DataSet({'x': [[1, 2], [2, 3, 4], [3]], 'y': [4, 5, 6]}) | ds = DataSet({'x': [[1, 2], [2, 3, 4], [3]], 'y': [4, 5, 6]}) | ||||
ds.add_seq_len('x') | ds.add_seq_len('x') | ||||
print(ds) | print(ds) | ||||
def test_set_target(self): | |||||
ds = DataSet({'x': [[1, 2], [2, 3, 4], [3]], 'y': [4, 5, 6]}) | |||||
ds.set_target('x') | |||||
class TestFieldArrayInit: | class TestFieldArrayInit: | ||||
""" | """ | ||||
@@ -19,7 +19,7 @@ if _NEED_IMPORT_PADDLE: | |||||
import paddle | import paddle | ||||
import paddle.distributed as dist | import paddle.distributed as dist | ||||
@pytest.mark.paddle | |||||
@pytest.mark.paddledist | |||||
class TestDistUtilsTools: | class TestDistUtilsTools: | ||||
""" | """ | ||||
测试一些工具函数 | 测试一些工具函数 | ||||
@@ -79,14 +79,13 @@ class TestDistUtilsTools: | |||||
assert res["int"] == paddle_dict["int"] | assert res["int"] == paddle_dict["int"] | ||||
assert res["string"] == paddle_dict["string"] | assert res["string"] == paddle_dict["string"] | ||||
@pytest.mark.paddle | |||||
@pytest.mark.paddledist | |||||
class TestAllGatherAndBroadCast: | class TestAllGatherAndBroadCast: | ||||
@classmethod | @classmethod | ||||
def setup_class(cls): | def setup_class(cls): | ||||
devices = [0,1,2] | devices = [0,1,2] | ||||
output_from_new_proc = "only_error" | |||||
output_from_new_proc = "all" | |||||
launcher = FleetLauncher(devices=devices, output_from_new_proc=output_from_new_proc) | launcher = FleetLauncher(devices=devices, output_from_new_proc=output_from_new_proc) | ||||
cls.local_rank = int(os.getenv("PADDLE_RANK_IN_NODE", "0")) | cls.local_rank = int(os.getenv("PADDLE_RANK_IN_NODE", "0")) | ||||
@@ -39,7 +39,7 @@ def generate_driver(num_labels, feature_dimension, device=[0,1], fp16=False, out | |||||
# | # | ||||
############################################################################ | ############################################################################ | ||||
@pytest.mark.paddle | |||||
@pytest.mark.paddledist | |||||
class TestFleetDriverFunction: | class TestFleetDriverFunction: | ||||
""" | """ | ||||
测试 PaddleFleetDriver 一些简单函数的测试类,基本都是测试能否运行、是否存在 import 错误等问题 | 测试 PaddleFleetDriver 一些简单函数的测试类,基本都是测试能否运行、是否存在 import 错误等问题 | ||||
@@ -147,7 +147,7 @@ class TestFleetDriverFunction: | |||||
# | # | ||||
############################################################################ | ############################################################################ | ||||
@pytest.mark.paddle | |||||
@pytest.mark.paddledist | |||||
class TestSetDistReproDataloader: | class TestSetDistReproDataloader: | ||||
@classmethod | @classmethod | ||||
@@ -521,7 +521,7 @@ class TestSetDistReproDataloader: | |||||
# | # | ||||
############################################################################ | ############################################################################ | ||||
@pytest.mark.paddle | |||||
@pytest.mark.paddledist | |||||
class TestSaveLoad: | class TestSaveLoad: | ||||
""" | """ | ||||
测试多卡情况下 save 和 load 相关函数的表现 | 测试多卡情况下 save 和 load 相关函数的表现 | ||||
@@ -552,22 +552,17 @@ def generate_random_driver(features, labels, fp16=False, device="cpu"): | |||||
return driver | return driver | ||||
@pytest.fixture | |||||
def prepare_test_save_load(): | |||||
dataset = PaddleRandomMaxDataset(40, 10) | |||||
dataloader = DataLoader(dataset, batch_size=4) | |||||
driver1, driver2 = generate_random_driver(10, 10), generate_random_driver(10, 10) | |||||
return driver1, driver2, dataloader | |||||
@pytest.mark.paddle | @pytest.mark.paddle | ||||
@pytest.mark.parametrize("only_state_dict", ([True, False])) | @pytest.mark.parametrize("only_state_dict", ([True, False])) | ||||
def test_save_and_load_model(prepare_test_save_load, only_state_dict): | |||||
def test_save_and_load_model(only_state_dict): | |||||
""" | """ | ||||
测试 save_model 和 load_model 函数 | 测试 save_model 和 load_model 函数 | ||||
""" | """ | ||||
try: | try: | ||||
path = "model" | path = "model" | ||||
driver1, driver2, dataloader = prepare_test_save_load | |||||
dataset = PaddleRandomMaxDataset(40, 10) | |||||
dataloader = DataLoader(dataset, batch_size=4) | |||||
driver1, driver2 = generate_random_driver(10, 10, device="gpu"), generate_random_driver(10, 10, device="gpu") | |||||
if only_state_dict: | if only_state_dict: | ||||
driver1.save_model(path, only_state_dict) | driver1.save_model(path, only_state_dict) | ||||
@@ -1,8 +1,6 @@ | |||||
import os | |||||
import pytest | import pytest | ||||
from fastNLP.core.drivers.paddle_driver.utils import ( | from fastNLP.core.drivers.paddle_driver.utils import ( | ||||
get_device_from_visible, | |||||
replace_batch_sampler, | replace_batch_sampler, | ||||
replace_sampler, | replace_sampler, | ||||
) | ) | ||||
@@ -14,24 +12,6 @@ if _NEED_IMPORT_PADDLE: | |||||
from tests.helpers.datasets.paddle_data import PaddleNormalDataset | from tests.helpers.datasets.paddle_data import PaddleNormalDataset | ||||
@pytest.mark.parametrize( | |||||
("user_visible_devices, cuda_visible_devices, device, output_type, correct"), | |||||
( | |||||
("0,1,2,3,4,5,6,7", "0", "cpu", str, "cpu"), | |||||
("0,1,2,3,4,5,6,7", "0", "cpu", int, "cpu"), | |||||
("0,1,2,3,4,5,6,7", "3,4,5", "gpu:4", int, 1), | |||||
("0,1,2,3,4,5,6,7", "3,4,5", "gpu:5", str, "gpu:2"), | |||||
("3,4,5,6", "3,5", 0, int, 0), | |||||
("3,6,7,8", "6,7,8", "gpu:2", str, "gpu:1"), | |||||
) | |||||
) | |||||
@pytest.mark.paddle | |||||
def test_get_device_from_visible_str(user_visible_devices, cuda_visible_devices, device, output_type, correct): | |||||
os.environ["CUDA_VISIBLE_DEVICES"] = cuda_visible_devices | |||||
os.environ["USER_CUDA_VISIBLE_DEVICES"] = user_visible_devices | |||||
res = get_device_from_visible(device, output_type) | |||||
assert res == correct | |||||
@pytest.mark.paddle | @pytest.mark.paddle | ||||
def test_replace_batch_sampler(): | def test_replace_batch_sampler(): | ||||
dataset = PaddleNormalDataset(10) | dataset = PaddleNormalDataset(10) | ||||
@@ -545,22 +545,17 @@ def generate_random_driver(features, labels, fp16=False, device="cpu"): | |||||
return driver | return driver | ||||
@pytest.fixture | |||||
def prepare_test_save_load(): | |||||
dataset = TorchArgMaxDataset(10, 40) | |||||
dataloader = DataLoader(dataset, batch_size=4) | |||||
driver1, driver2 = generate_random_driver(10, 10), generate_random_driver(10, 10) | |||||
return driver1, driver2, dataloader | |||||
@pytest.mark.torch | @pytest.mark.torch | ||||
@pytest.mark.parametrize("only_state_dict", ([True, False])) | @pytest.mark.parametrize("only_state_dict", ([True, False])) | ||||
def test_save_and_load_model(prepare_test_save_load, only_state_dict): | |||||
def test_save_and_load_model(only_state_dict): | |||||
""" | """ | ||||
测试 save_model 和 load_model 函数 | 测试 save_model 和 load_model 函数 | ||||
""" | """ | ||||
try: | try: | ||||
path = "model" | path = "model" | ||||
driver1, driver2, dataloader = prepare_test_save_load | |||||
dataset = TorchArgMaxDataset(10, 40) | |||||
dataloader = DataLoader(dataset, batch_size=4) | |||||
driver1, driver2 = generate_random_driver(10, 10), generate_random_driver(10, 10) | |||||
driver1.save_model(path, only_state_dict) | driver1.save_model(path, only_state_dict) | ||||
driver2.load_model(path, only_state_dict) | driver2.load_model(path, only_state_dict) | ||||
@@ -1,10 +1,40 @@ | |||||
import os | |||||
import pytest | import pytest | ||||
from fastNLP.core.utils.paddle_utils import paddle_to, paddle_move_data_to_device | |||||
from fastNLP.core.utils.paddle_utils import get_device_from_visible, paddle_to, paddle_move_data_to_device | |||||
from fastNLP.envs.imports import _NEED_IMPORT_PADDLE | from fastNLP.envs.imports import _NEED_IMPORT_PADDLE | ||||
if _NEED_IMPORT_PADDLE: | if _NEED_IMPORT_PADDLE: | ||||
import paddle | import paddle | ||||
@pytest.mark.parametrize( | |||||
("user_visible_devices, cuda_visible_devices, device, output_type, correct"), | |||||
( | |||||
("0,1,2,3,4,5,6,7", "0", "cpu", str, "cpu"), | |||||
("0,1,2,3,4,5,6,7", "0", "cpu", int, "cpu"), | |||||
("0,1,2,3,4,5,6,7", "3,4,5", "gpu:4", int, 1), | |||||
("0,1,2,3,4,5,6,7", "3,4,5", "gpu:5", str, "gpu:2"), | |||||
("3,4,5,6", "3,5", 0, int, 0), | |||||
("3,6,7,8", "6,7,8", "gpu:2", str, "gpu:1"), | |||||
) | |||||
) | |||||
@pytest.mark.paddle | |||||
def test_get_device_from_visible(user_visible_devices, cuda_visible_devices, device, output_type, correct): | |||||
_cuda_visible_devices = os.getenv("CUDA_VISIBLE_DEVICES") | |||||
_user_visible_devices = os.getenv("USER_CUDA_VISIBLE_DEVICES") | |||||
os.environ["CUDA_VISIBLE_DEVICES"] = cuda_visible_devices | |||||
os.environ["USER_CUDA_VISIBLE_DEVICES"] = user_visible_devices | |||||
res = get_device_from_visible(device, output_type) | |||||
assert res == correct | |||||
# 还原环境变量 | |||||
if _cuda_visible_devices is None: | |||||
del os.environ["CUDA_VISIBLE_DEVICES"] | |||||
else: | |||||
os.environ["CUDA_VISIBLE_DEVICES"] = _cuda_visible_devices | |||||
if _user_visible_devices is None: | |||||
del os.environ["USER_CUDA_VISIBLE_DEVICES"] | |||||
else: | |||||
os.environ["USER_CUDA_VISIBLE_DEVICES"] = _user_visible_devices | |||||
############################################################################ | ############################################################################ | ||||
# | # | ||||
@@ -22,12 +52,6 @@ class TestPaddleToDevice: | |||||
assert res.place.gpu_device_id() == 0 | assert res.place.gpu_device_id() == 0 | ||||
res = paddle_to(tensor, "cpu") | res = paddle_to(tensor, "cpu") | ||||
assert res.place.is_cpu_place() | assert res.place.is_cpu_place() | ||||
res = paddle_to(tensor, "gpu:2") | |||||
assert res.place.is_gpu_place() | |||||
assert res.place.gpu_device_id() == 2 | |||||
res = paddle_to(tensor, "gpu:1") | |||||
assert res.place.is_gpu_place() | |||||
assert res.place.gpu_device_id() == 1 | |||||
############################################################################ | ############################################################################ | ||||
# | # | ||||
@@ -64,28 +88,18 @@ class TestPaddleMoveDataToDevice: | |||||
res = paddle_move_data_to_device(paddle_tensor, device="gpu:0", data_device=None) | res = paddle_move_data_to_device(paddle_tensor, device="gpu:0", data_device=None) | ||||
self.check_gpu(res, 0) | self.check_gpu(res, 0) | ||||
res = paddle_move_data_to_device(paddle_tensor, device="gpu:1", data_device=None) | |||||
self.check_gpu(res, 1) | |||||
res = paddle_move_data_to_device(paddle_tensor, device="gpu:0", data_device="cpu") | res = paddle_move_data_to_device(paddle_tensor, device="gpu:0", data_device="cpu") | ||||
self.check_gpu(res, 0) | self.check_gpu(res, 0) | ||||
res = paddle_move_data_to_device(paddle_tensor, device=None, data_device="gpu:0") | res = paddle_move_data_to_device(paddle_tensor, device=None, data_device="gpu:0") | ||||
self.check_gpu(res, 0) | self.check_gpu(res, 0) | ||||
res = paddle_move_data_to_device(paddle_tensor, device=None, data_device="gpu:1") | |||||
self.check_gpu(res, 1) | |||||
def test_list_transfer(self): | def test_list_transfer(self): | ||||
""" | """ | ||||
测试张量列表的迁移 | 测试张量列表的迁移 | ||||
""" | """ | ||||
paddle_list = [paddle.rand((6, 4, 2)) for i in range(10)] | paddle_list = [paddle.rand((6, 4, 2)) for i in range(10)] | ||||
res = paddle_move_data_to_device(paddle_list, device=None, data_device="gpu:1") | |||||
assert isinstance(res, list) | |||||
for r in res: | |||||
self.check_gpu(r, 1) | |||||
res = paddle_move_data_to_device(paddle_list, device="cpu", data_device="gpu:1") | res = paddle_move_data_to_device(paddle_list, device="cpu", data_device="gpu:1") | ||||
assert isinstance(res, list) | assert isinstance(res, list) | ||||
@@ -97,11 +111,6 @@ class TestPaddleMoveDataToDevice: | |||||
for r in res: | for r in res: | ||||
self.check_gpu(r, 0) | self.check_gpu(r, 0) | ||||
res = paddle_move_data_to_device(paddle_list, device="gpu:1", data_device="cpu") | |||||
assert isinstance(res, list) | |||||
for r in res: | |||||
self.check_gpu(r, 1) | |||||
def test_tensor_tuple_transfer(self): | def test_tensor_tuple_transfer(self): | ||||
""" | """ | ||||
测试张量元组的迁移 | 测试张量元组的迁移 | ||||
@@ -109,10 +118,6 @@ class TestPaddleMoveDataToDevice: | |||||
paddle_list = [paddle.rand((6, 4, 2)) for i in range(10)] | paddle_list = [paddle.rand((6, 4, 2)) for i in range(10)] | ||||
paddle_tuple = tuple(paddle_list) | paddle_tuple = tuple(paddle_list) | ||||
res = paddle_move_data_to_device(paddle_tuple, device=None, data_device="gpu:1") | |||||
assert isinstance(res, tuple) | |||||
for r in res: | |||||
self.check_gpu(r, 1) | |||||
res = paddle_move_data_to_device(paddle_tuple, device="cpu", data_device="gpu:1") | res = paddle_move_data_to_device(paddle_tuple, device="cpu", data_device="gpu:1") | ||||
assert isinstance(res, tuple) | assert isinstance(res, tuple) | ||||
@@ -124,11 +129,6 @@ class TestPaddleMoveDataToDevice: | |||||
for r in res: | for r in res: | ||||
self.check_gpu(r, 0) | self.check_gpu(r, 0) | ||||
res = paddle_move_data_to_device(paddle_tuple, device="gpu:1", data_device="cpu") | |||||
assert isinstance(res, tuple) | |||||
for r in res: | |||||
self.check_gpu(r, 1) | |||||
def test_dict_transfer(self): | def test_dict_transfer(self): | ||||
""" | """ | ||||
测试字典结构的迁移 | 测试字典结构的迁移 | ||||
@@ -173,20 +173,6 @@ class TestPaddleMoveDataToDevice: | |||||
self.check_gpu(t, 0) | self.check_gpu(t, 0) | ||||
self.check_gpu(res["dict"]["tensor"], 0) | self.check_gpu(res["dict"]["tensor"], 0) | ||||
res = paddle_move_data_to_device(paddle_dict, device=None, data_device="gpu:1") | |||||
assert isinstance(res, dict) | |||||
self.check_gpu(res["tensor"], 1) | |||||
assert isinstance(res["list"], list) | |||||
for t in res["list"]: | |||||
self.check_gpu(t, 1) | |||||
assert isinstance(res["int"], int) | |||||
assert isinstance(res["string"], str) | |||||
assert isinstance(res["dict"], dict) | |||||
assert isinstance(res["dict"]["list"], list) | |||||
for t in res["dict"]["list"]: | |||||
self.check_gpu(t, 1) | |||||
self.check_gpu(res["dict"]["tensor"], 1) | |||||
res = paddle_move_data_to_device(paddle_dict, device="cpu", data_device="gpu:0") | res = paddle_move_data_to_device(paddle_dict, device="cpu", data_device="gpu:0") | ||||
assert isinstance(res, dict) | assert isinstance(res, dict) | ||||
self.check_cpu(res["tensor"]) | self.check_cpu(res["tensor"]) | ||||
@@ -2,5 +2,6 @@ | |||||
markers = | markers = | ||||
torch | torch | ||||
paddle | paddle | ||||
paddledist | |||||
jittor | jittor | ||||
torchpaddle | torchpaddle |
@@ -0,0 +1,7 @@ | |||||
,SentenceId,Sentence,Sentiment | |||||
0,1,"['a', 'series', 'of', 'escapades', 'demonstrating', 'the', 'adage', 'that', 'what', 'is', 'good', 'for', 'the', 'goose', 'is', 'also', 'good', 'for', 'the', 'gander', ',', 'some', 'of', 'which', 'occasionally', 'amuses', 'but', 'none', 'of', 'which', 'amounts', 'to', 'much', 'of', 'a', 'story', '.']",negative | |||||
1,2,"['this', 'quiet', ',', 'introspective', 'and', 'entertaining', 'independent', 'is', 'worth', 'seeking', '.']",positive | |||||
2,3,"['even', 'fans', 'of', 'ismail', 'merchant', ""'s"", 'work', ',', 'i', 'suspect', ',', 'would', 'have', 'a', 'hard', 'time', 'sitting', 'through', 'this', 'one', '.']",negative | |||||
3,4,"['a', 'positively', 'thrilling', 'combination', 'of', 'ethnography', 'and', 'all', 'the', 'intrigue', ',', 'betrayal', ',', 'deceit', 'and', 'murder', 'of', 'a', 'shakespearean', 'tragedy', 'or', 'a', 'juicy', 'soap', 'opera', '.']",neutral | |||||
4,5,"['a', 'comedy-drama', 'of', 'nearly', 'epic', 'proportions', 'rooted', 'in', 'a', 'sincere', 'performance', 'by', 'the', 'title', 'character', 'undergoing', 'midlife', 'crisis', '.']",positive | |||||
5,6,"['the', 'importance', 'of', 'being', 'earnest', ',', 'so', 'thick', 'with', 'wit', 'it', 'plays', 'like', 'a', 'reading', 'from', 'bartlett', ""'s"", 'familiar', 'quotations']",neutral |
@@ -0,0 +1,7 @@ | |||||
SentenceId Sentence Sentiment | |||||
1 A series of escapades demonstrating the adage that what is good for the goose is also good for the gander , some of which occasionally amuses but none of which amounts to much of a story . negative | |||||
2 This quiet , introspective and entertaining independent is worth seeking . positive | |||||
3 Even fans of Ismail Merchant 's work , I suspect , would have a hard time sitting through this one . negative | |||||
4 A positively thrilling combination of ethnography and all the intrigue , betrayal , deceit and murder of a Shakespearean tragedy or a juicy soap opera . neutral | |||||
5 A comedy-drama of nearly epic proportions rooted in a sincere performance by the title character undergoing midlife crisis . positive | |||||
6 The Importance of Being Earnest , so thick with wit it plays like a reading from Bartlett 's Familiar Quotations neutral |
@@ -153,7 +153,7 @@ | |||||
"name": "stdout", | "name": "stdout", | ||||
"output_type": "stream", | "output_type": "stream", | ||||
"text": [ | "text": [ | ||||
"1969418794120 1971237588872\n", | |||||
"2438703969992 2438374526920\n", | |||||
"+-----+------------------------+------------------------+-----+\n", | "+-----+------------------------+------------------------+-----+\n", | ||||
"| idx | sentence | words | num |\n", | "| idx | sentence | words | num |\n", | ||||
"+-----+------------------------+------------------------+-----+\n", | "+-----+------------------------+------------------------+-----+\n", | ||||
@@ -198,7 +198,7 @@ | |||||
"name": "stdout", | "name": "stdout", | ||||
"output_type": "stream", | "output_type": "stream", | ||||
"text": [ | "text": [ | ||||
"1971237588872 1971237588872\n", | |||||
"2438374526920 2438374526920\n", | |||||
"+-----+------------------------+------------------------+-----+\n", | "+-----+------------------------+------------------------+-----+\n", | ||||
"| idx | sentence | words | num |\n", | "| idx | sentence | words | num |\n", | ||||
"+-----+------------------------+------------------------+-----+\n", | "+-----+------------------------+------------------------+-----+\n", | ||||
@@ -774,9 +774,9 @@ | |||||
{ | { | ||||
"data": { | "data": { | ||||
"text/plain": [ | "text/plain": [ | ||||
"{'sentence': <fastNLP.core.dataset.field.FieldArray at 0x1ca8a879d08>,\n", | |||||
" 'words': <fastNLP.core.dataset.field.FieldArray at 0x1ca8a879d88>,\n", | |||||
" 'num': <fastNLP.core.dataset.field.FieldArray at 0x1ca8a879e08>}" | |||||
"{'sentence': <fastNLP.core.dataset.field.FieldArray at 0x237ce26d388>,\n", | |||||
" 'words': <fastNLP.core.dataset.field.FieldArray at 0x237ce26d408>,\n", | |||||
" 'num': <fastNLP.core.dataset.field.FieldArray at 0x237ce26d488>}" | |||||
] | ] | ||||
}, | }, | ||||
"execution_count": 15, | "execution_count": 15, | ||||
@@ -923,7 +923,8 @@ | |||||
"output_type": "stream", | "output_type": "stream", | ||||
"text": [ | "text": [ | ||||
"5 Counter({'生活': 1, '就像': 1, '海洋': 1})\n", | "5 Counter({'生活': 1, '就像': 1, '海洋': 1})\n", | ||||
"6 Counter({'生活': 1, '就像': 1, '海洋': 1, '只有': 1})\n" | |||||
"6 Counter({'生活': 1, '就像': 1, '海洋': 1, '只有': 1})\n", | |||||
"6 {'<pad>': 0, '<unk>': 1, '生活': 2, '就像': 3, '海洋': 4, '只有': 5}\n" | |||||
] | ] | ||||
} | } | ||||
], | ], | ||||
@@ -931,7 +932,8 @@ | |||||
"vocab.add_word_lst(['生活', '就像', '海洋'])\n", | "vocab.add_word_lst(['生活', '就像', '海洋'])\n", | ||||
"print(len(vocab), vocab.word_count)\n", | "print(len(vocab), vocab.word_count)\n", | ||||
"vocab.add_word('只有')\n", | "vocab.add_word('只有')\n", | ||||
"print(len(vocab), vocab.word_count)" | |||||
"print(len(vocab), vocab.word_count)\n", | |||||
"print(len(vocab), vocab.word2idx)" | |||||
] | ] | ||||
}, | }, | ||||
{ | { | ||||
@@ -959,7 +961,6 @@ | |||||
"<pad> 0\n", | "<pad> 0\n", | ||||
"<unk> 1\n", | "<unk> 1\n", | ||||
"生活 2\n", | "生活 2\n", | ||||
"只有 5\n", | |||||
"彼岸 1 False\n" | "彼岸 1 False\n" | ||||
] | ] | ||||
} | } | ||||
@@ -968,7 +969,6 @@ | |||||
"print(vocab.to_word(0), vocab.to_index('<pad>'))\n", | "print(vocab.to_word(0), vocab.to_index('<pad>'))\n", | ||||
"print(vocab.to_word(1), vocab.to_index('<unk>'))\n", | "print(vocab.to_word(1), vocab.to_index('<unk>'))\n", | ||||
"print(vocab.to_word(2), vocab.to_index('生活'))\n", | "print(vocab.to_word(2), vocab.to_index('生活'))\n", | ||||
"print(vocab.to_word(5), vocab.to_index('只有'))\n", | |||||
"print('彼岸', vocab.to_index('彼岸'), vocab.has_word('彼岸'))" | "print('彼岸', vocab.to_index('彼岸'), vocab.has_word('彼岸'))" | ||||
] | ] | ||||
}, | }, | ||||
@@ -979,7 +979,9 @@ | |||||
"source": [ | "source": [ | ||||
"**`vocabulary`允许反复添加相同单词**,**可以通过`word_count`方法看到相应单词被添加的次数**\n", | "**`vocabulary`允许反复添加相同单词**,**可以通过`word_count`方法看到相应单词被添加的次数**\n", | ||||
"\n", | "\n", | ||||
"  但其中没有`<unk>`和`<pad>`,`vocabulary`的全部变量与函数可以通过`dir(vocabulary)`查询" | |||||
"  但其中没有`<unk>`和`<pad>`,`vocabulary`的全部变量与函数可以通过`dir(vocabulary)`查询\n", | |||||
"\n", | |||||
"  注:**使用`add_word_lst`添加单词**,**单词对应序号不会动态调整**,**使用`dataset`添加单词的情况不同**" | |||||
] | ] | ||||
}, | }, | ||||
{ | { | ||||
@@ -992,15 +994,19 @@ | |||||
"name": "stdout", | "name": "stdout", | ||||
"output_type": "stream", | "output_type": "stream", | ||||
"text": [ | "text": [ | ||||
"13 Counter({'生活': 2, '就像': 2, '海洋': 2, '只有': 2, '意志': 1, '坚强的': 1, '人': 1, '才': 1, '能': 1, '到达': 1, '彼岸': 1})\n", | |||||
"彼岸 12 True\n" | |||||
"生活 2\n", | |||||
"彼岸 12 True\n", | |||||
"13 Counter({'人': 4, '生活': 2, '就像': 2, '海洋': 2, '只有': 2, '意志': 1, '坚强的': 1, '才': 1, '能': 1, '到达': 1, '彼岸': 1})\n", | |||||
"13 {'<pad>': 0, '<unk>': 1, '生活': 2, '就像': 3, '海洋': 4, '只有': 5, '人': 6, '意志': 7, '坚强的': 8, '才': 9, '能': 10, '到达': 11, '彼岸': 12}\n" | |||||
] | ] | ||||
} | } | ||||
], | ], | ||||
"source": [ | "source": [ | ||||
"vocab.add_word_lst(['生活', '就像', '海洋', '只有', '意志', '坚强的', '人', '才', '能', '到达', '彼岸'])\n", | |||||
"vocab.add_word_lst(['生活', '就像', '海洋', '只有', '意志', '坚强的', '人', '人', '人', '人', '才', '能', '到达', '彼岸'])\n", | |||||
"print(vocab.to_word(2), vocab.to_index('生活'))\n", | |||||
"print('彼岸', vocab.to_index('彼岸'), vocab.has_word('彼岸'))\n", | |||||
"print(len(vocab), vocab.word_count)\n", | "print(len(vocab), vocab.word_count)\n", | ||||
"print('彼岸', vocab.to_index('彼岸'), vocab.has_word('彼岸'))" | |||||
"print(len(vocab), vocab.word2idx)" | |||||
] | ] | ||||
}, | }, | ||||
{ | { | ||||
@@ -1082,52 +1088,440 @@ | |||||
"## 3 dataset 和 vocabulary 的组合使用\n", | "## 3 dataset 和 vocabulary 的组合使用\n", | ||||
" \n", | " \n", | ||||
"### 3.1 从 dataframe 中加载 dataset\n", | "### 3.1 从 dataframe 中加载 dataset\n", | ||||
"\n" | |||||
"\n", | |||||
"以下通过 [NLP-beginner](https://github.com/FudanNLP/nlp-beginner) 实践一中 [Rotten Tomatoes 影评数据集](https://www.kaggle.com/c/sentiment-analysis-on-movie-reviews) 的部分训练数据组成`test4dataset.tsv`文件\n", | |||||
"\n", | |||||
"  介绍如何使用`dataset`、`vocabulary`简单加载并处理数据集,首先使用`pandas`模块,读取原始数据的`dataframe`" | |||||
] | |||||
}, | |||||
{ | |||||
"cell_type": "code", | |||||
"execution_count": 24, | |||||
"id": "3dbd985d", | |||||
"metadata": {}, | |||||
"outputs": [ | |||||
{ | |||||
"data": { | |||||
"text/html": [ | |||||
"<div>\n", | |||||
"<style scoped>\n", | |||||
" .dataframe tbody tr th:only-of-type {\n", | |||||
" vertical-align: middle;\n", | |||||
" }\n", | |||||
"\n", | |||||
" .dataframe tbody tr th {\n", | |||||
" vertical-align: top;\n", | |||||
" }\n", | |||||
"\n", | |||||
" .dataframe thead th {\n", | |||||
" text-align: right;\n", | |||||
" }\n", | |||||
"</style>\n", | |||||
"<table border=\"1\" class=\"dataframe\">\n", | |||||
" <thead>\n", | |||||
" <tr style=\"text-align: right;\">\n", | |||||
" <th></th>\n", | |||||
" <th>SentenceId</th>\n", | |||||
" <th>Sentence</th>\n", | |||||
" <th>Sentiment</th>\n", | |||||
" </tr>\n", | |||||
" </thead>\n", | |||||
" <tbody>\n", | |||||
" <tr>\n", | |||||
" <th>0</th>\n", | |||||
" <td>1</td>\n", | |||||
" <td>A series of escapades demonstrating the adage ...</td>\n", | |||||
" <td>negative</td>\n", | |||||
" </tr>\n", | |||||
" <tr>\n", | |||||
" <th>1</th>\n", | |||||
" <td>2</td>\n", | |||||
" <td>This quiet , introspective and entertaining in...</td>\n", | |||||
" <td>positive</td>\n", | |||||
" </tr>\n", | |||||
" <tr>\n", | |||||
" <th>2</th>\n", | |||||
" <td>3</td>\n", | |||||
" <td>Even fans of Ismail Merchant 's work , I suspe...</td>\n", | |||||
" <td>negative</td>\n", | |||||
" </tr>\n", | |||||
" <tr>\n", | |||||
" <th>3</th>\n", | |||||
" <td>4</td>\n", | |||||
" <td>A positively thrilling combination of ethnogra...</td>\n", | |||||
" <td>neutral</td>\n", | |||||
" </tr>\n", | |||||
" <tr>\n", | |||||
" <th>4</th>\n", | |||||
" <td>5</td>\n", | |||||
" <td>A comedy-drama of nearly epic proportions root...</td>\n", | |||||
" <td>positive</td>\n", | |||||
" </tr>\n", | |||||
" <tr>\n", | |||||
" <th>5</th>\n", | |||||
" <td>6</td>\n", | |||||
" <td>The Importance of Being Earnest , so thick wit...</td>\n", | |||||
" <td>neutral</td>\n", | |||||
" </tr>\n", | |||||
" </tbody>\n", | |||||
"</table>\n", | |||||
"</div>" | |||||
], | |||||
"text/plain": [ | |||||
" SentenceId Sentence Sentiment\n", | |||||
"0 1 A series of escapades demonstrating the adage ... negative\n", | |||||
"1 2 This quiet , introspective and entertaining in... positive\n", | |||||
"2 3 Even fans of Ismail Merchant 's work , I suspe... negative\n", | |||||
"3 4 A positively thrilling combination of ethnogra... neutral\n", | |||||
"4 5 A comedy-drama of nearly epic proportions root... positive\n", | |||||
"5 6 The Importance of Being Earnest , so thick wit... neutral" | |||||
] | |||||
}, | |||||
"execution_count": 24, | |||||
"metadata": {}, | |||||
"output_type": "execute_result" | |||||
} | |||||
], | |||||
"source": [ | |||||
"import pandas as pd\n", | |||||
"\n", | |||||
"df = pd.read_csv('./data/test4dataset.tsv', sep='\\t')\n", | |||||
"df" | |||||
] | ] | ||||
}, | }, | ||||
{ | { | ||||
"cell_type": "markdown", | "cell_type": "markdown", | ||||
"id": "89059713", | |||||
"id": "919ab350", | |||||
"metadata": {}, | "metadata": {}, | ||||
"source": [] | |||||
"source": [ | |||||
"接着,通过`dataset`中的`from_pandas`方法填充数据集,并使用`apply_more`方法对文本进行分词操作" | |||||
] | |||||
}, | }, | ||||
{ | { | ||||
"cell_type": "code", | "cell_type": "code", | ||||
"execution_count": null, | |||||
"id": "3dbd985d", | |||||
"execution_count": 25, | |||||
"id": "4f634586", | |||||
"metadata": {}, | "metadata": {}, | ||||
"outputs": [], | |||||
"source": [] | |||||
"outputs": [ | |||||
{ | |||||
"data": { | |||||
"text/html": [ | |||||
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n" | |||||
], | |||||
"text/plain": [] | |||||
}, | |||||
"metadata": {}, | |||||
"output_type": "display_data" | |||||
}, | |||||
{ | |||||
"data": { | |||||
"text/html": [ | |||||
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n" | |||||
], | |||||
"text/plain": [] | |||||
}, | |||||
"metadata": {}, | |||||
"output_type": "display_data" | |||||
}, | |||||
{ | |||||
"data": { | |||||
"text/html": [ | |||||
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n", | |||||
"</pre>\n" | |||||
], | |||||
"text/plain": [ | |||||
"\n" | |||||
] | |||||
}, | |||||
"metadata": {}, | |||||
"output_type": "display_data" | |||||
}, | |||||
{ | |||||
"name": "stdout", | |||||
"output_type": "stream", | |||||
"text": [ | |||||
"+------------+------------------------------+-----------+\n", | |||||
"| SentenceId | Sentence | Sentiment |\n", | |||||
"+------------+------------------------------+-----------+\n", | |||||
"| 1 | ['a', 'series', 'of', 'es... | negative |\n", | |||||
"| 2 | ['this', 'quiet', ',', 'i... | positive |\n", | |||||
"| 3 | ['even', 'fans', 'of', 'i... | negative |\n", | |||||
"| 4 | ['a', 'positively', 'thri... | neutral |\n", | |||||
"| 5 | ['a', 'comedy-drama', 'of... | positive |\n", | |||||
"| 6 | ['the', 'importance', 'of... | neutral |\n", | |||||
"+------------+------------------------------+-----------+\n" | |||||
] | |||||
} | |||||
], | |||||
"source": [ | |||||
"from fastNLP.core.dataset import DataSet\n", | |||||
"\n", | |||||
"dataset = DataSet()\n", | |||||
"dataset = dataset.from_pandas(df)\n", | |||||
"dataset.apply_more(lambda ins:{'SentenceId': ins['SentenceId'], \n", | |||||
" 'Sentence': ins['Sentence'].lower().split(), 'Sentiment': ins['Sentiment']})\n", | |||||
"print(dataset)" | |||||
] | |||||
}, | |||||
{ | |||||
"cell_type": "markdown", | |||||
"id": "5c1ae192", | |||||
"metadata": {}, | |||||
"source": [ | |||||
"  如果需要保存中间结果,也可以使用`dataset`的`to_csv`方法,生成`.csv`或`.tsv`文件" | |||||
] | |||||
}, | }, | ||||
{ | { | ||||
"cell_type": "code", | "cell_type": "code", | ||||
"execution_count": null, | |||||
"id": "4f634586", | |||||
"execution_count": 26, | |||||
"id": "46722efc", | |||||
"metadata": {}, | "metadata": {}, | ||||
"outputs": [], | "outputs": [], | ||||
"source": [] | |||||
"source": [ | |||||
"dataset.to_csv('./data/test4dataset.csv')" | |||||
] | |||||
}, | }, | ||||
{ | { | ||||
"cell_type": "markdown", | "cell_type": "markdown", | ||||
"id": "5ba13989", | "id": "5ba13989", | ||||
"metadata": {}, | "metadata": {}, | ||||
"source": [ | "source": [ | ||||
"### 3.2 从 dataset 中获取 vocabulary" | |||||
"### 3.2 从 dataset 中获取 vocabulary\n", | |||||
"\n", | |||||
"然后,初始化`vocabulary`,使用`vocabulary`中的`from_dataset`方法,从`dataset`的指定字段中\n", | |||||
"\n", | |||||
"  获取字段中的所有元素,然后编号;如果指定字段是个列表,则针对字段中所有列表包含的元素编号\n", | |||||
"\n", | |||||
"  注:**使用`dataset`添加单词**,**不同于`add_word_list`**,**单词被添加次数越多**,**序号越靠前**,例如案例中的`a`" | |||||
] | ] | ||||
}, | }, | ||||
{ | { | ||||
"cell_type": "code", | "cell_type": "code", | ||||
"execution_count": null, | |||||
"execution_count": 27, | |||||
"id": "a2de615b", | "id": "a2de615b", | ||||
"metadata": {}, | "metadata": {}, | ||||
"outputs": [], | |||||
"source": [] | |||||
"outputs": [ | |||||
{ | |||||
"data": { | |||||
"text/html": [ | |||||
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n" | |||||
], | |||||
"text/plain": [] | |||||
}, | |||||
"metadata": {}, | |||||
"output_type": "display_data" | |||||
}, | |||||
{ | |||||
"data": { | |||||
"text/html": [ | |||||
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n" | |||||
], | |||||
"text/plain": [] | |||||
}, | |||||
"metadata": {}, | |||||
"output_type": "display_data" | |||||
}, | |||||
{ | |||||
"data": { | |||||
"text/html": [ | |||||
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n", | |||||
"</pre>\n" | |||||
], | |||||
"text/plain": [ | |||||
"\n" | |||||
] | |||||
}, | |||||
"metadata": {}, | |||||
"output_type": "display_data" | |||||
}, | |||||
{ | |||||
"name": "stdout", | |||||
"output_type": "stream", | |||||
"text": [ | |||||
"Counter({'a': 9, 'of': 9, ',': 7, 'the': 6, '.': 5, 'is': 3, 'and': 3, 'good': 2, 'for': 2, 'which': 2, 'this': 2, \"'s\": 2, 'series': 1, 'escapades': 1, 'demonstrating': 1, 'adage': 1, 'that': 1, 'what': 1, 'goose': 1, 'also': 1, 'gander': 1, 'some': 1, 'occasionally': 1, 'amuses': 1, 'but': 1, 'none': 1, 'amounts': 1, 'to': 1, 'much': 1, 'story': 1, 'quiet': 1, 'introspective': 1, 'entertaining': 1, 'independent': 1, 'worth': 1, 'seeking': 1, 'even': 1, 'fans': 1, 'ismail': 1, 'merchant': 1, 'work': 1, 'i': 1, 'suspect': 1, 'would': 1, 'have': 1, 'hard': 1, 'time': 1, 'sitting': 1, 'through': 1, 'one': 1, 'positively': 1, 'thrilling': 1, 'combination': 1, 'ethnography': 1, 'all': 1, 'intrigue': 1, 'betrayal': 1, 'deceit': 1, 'murder': 1, 'shakespearean': 1, 'tragedy': 1, 'or': 1, 'juicy': 1, 'soap': 1, 'opera': 1, 'comedy-drama': 1, 'nearly': 1, 'epic': 1, 'proportions': 1, 'rooted': 1, 'in': 1, 'sincere': 1, 'performance': 1, 'by': 1, 'title': 1, 'character': 1, 'undergoing': 1, 'midlife': 1, 'crisis': 1, 'importance': 1, 'being': 1, 'earnest': 1, 'so': 1, 'thick': 1, 'with': 1, 'wit': 1, 'it': 1, 'plays': 1, 'like': 1, 'reading': 1, 'from': 1, 'bartlett': 1, 'familiar': 1, 'quotations': 1}) \n", | |||||
"\n", | |||||
"{'<pad>': 0, '<unk>': 1, 'a': 2, 'of': 3, ',': 4, 'the': 5, '.': 6, 'is': 7, 'and': 8, 'good': 9, 'for': 10, 'which': 11, 'this': 12, \"'s\": 13, 'series': 14, 'escapades': 15, 'demonstrating': 16, 'adage': 17, 'that': 18, 'what': 19, 'goose': 20, 'also': 21, 'gander': 22, 'some': 23, 'occasionally': 24, 'amuses': 25, 'but': 26, 'none': 27, 'amounts': 28, 'to': 29, 'much': 30, 'story': 31, 'quiet': 32, 'introspective': 33, 'entertaining': 34, 'independent': 35, 'worth': 36, 'seeking': 37, 'even': 38, 'fans': 39, 'ismail': 40, 'merchant': 41, 'work': 42, 'i': 43, 'suspect': 44, 'would': 45, 'have': 46, 'hard': 47, 'time': 48, 'sitting': 49, 'through': 50, 'one': 51, 'positively': 52, 'thrilling': 53, 'combination': 54, 'ethnography': 55, 'all': 56, 'intrigue': 57, 'betrayal': 58, 'deceit': 59, 'murder': 60, 'shakespearean': 61, 'tragedy': 62, 'or': 63, 'juicy': 64, 'soap': 65, 'opera': 66, 'comedy-drama': 67, 'nearly': 68, 'epic': 69, 'proportions': 70, 'rooted': 71, 'in': 72, 'sincere': 73, 'performance': 74, 'by': 75, 'title': 76, 'character': 77, 'undergoing': 78, 'midlife': 79, 'crisis': 80, 'importance': 81, 'being': 82, 'earnest': 83, 'so': 84, 'thick': 85, 'with': 86, 'wit': 87, 'it': 88, 'plays': 89, 'like': 90, 'reading': 91, 'from': 92, 'bartlett': 93, 'familiar': 94, 'quotations': 95} \n", | |||||
"\n", | |||||
"Vocabulary(['a', 'series', 'of', 'escapades', 'demonstrating']...)\n" | |||||
] | |||||
} | |||||
], | |||||
"source": [ | |||||
"from fastNLP.core.vocabulary import Vocabulary\n", | |||||
"\n", | |||||
"vocab = Vocabulary()\n", | |||||
"vocab = vocab.from_dataset(dataset, field_name='Sentence')\n", | |||||
"print(vocab.word_count, '\\n')\n", | |||||
"print(vocab.word2idx, '\\n')\n", | |||||
"print(vocab)" | |||||
] | |||||
}, | |||||
{ | |||||
"cell_type": "markdown", | |||||
"id": "f0857ccb", | |||||
"metadata": {}, | |||||
"source": [ | |||||
"之后,**通过`vocabulary`的`index_dataset`方法**,**调整`dataset`中指定字段的元素**,**使用编号将之代替**\n", | |||||
"\n", | |||||
"  使用上述方法,可以将影评数据集中的单词序列转化为词编号序列,为接下来转化为词嵌入序列做准备" | |||||
] | |||||
}, | }, | ||||
{ | { | ||||
"cell_type": "code", | "cell_type": "code", | ||||
"execution_count": null, | |||||
"execution_count": 28, | |||||
"id": "2f9a04b2", | |||||
"metadata": {}, | |||||
"outputs": [ | |||||
{ | |||||
"data": { | |||||
"text/html": [ | |||||
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n" | |||||
], | |||||
"text/plain": [] | |||||
}, | |||||
"metadata": {}, | |||||
"output_type": "display_data" | |||||
}, | |||||
{ | |||||
"data": { | |||||
"text/html": [ | |||||
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n" | |||||
], | |||||
"text/plain": [] | |||||
}, | |||||
"metadata": {}, | |||||
"output_type": "display_data" | |||||
}, | |||||
{ | |||||
"data": { | |||||
"text/html": [ | |||||
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n", | |||||
"</pre>\n" | |||||
], | |||||
"text/plain": [ | |||||
"\n" | |||||
] | |||||
}, | |||||
"metadata": {}, | |||||
"output_type": "display_data" | |||||
}, | |||||
{ | |||||
"name": "stdout", | |||||
"output_type": "stream", | |||||
"text": [ | |||||
"+------------+------------------------------+-----------+\n", | |||||
"| SentenceId | Sentence | Sentiment |\n", | |||||
"+------------+------------------------------+-----------+\n", | |||||
"| 1 | [2, 14, 3, 15, 16, 5, 17,... | negative |\n", | |||||
"| 2 | [12, 32, 4, 33, 8, 34, 35... | positive |\n", | |||||
"| 3 | [38, 39, 3, 40, 41, 13, 4... | negative |\n", | |||||
"| 4 | [2, 52, 53, 54, 3, 55, 8,... | neutral |\n", | |||||
"| 5 | [2, 67, 3, 68, 69, 70, 71... | positive |\n", | |||||
"| 6 | [5, 81, 3, 82, 83, 4, 84,... | neutral |\n", | |||||
"+------------+------------------------------+-----------+\n" | |||||
] | |||||
} | |||||
], | |||||
"source": [ | |||||
"vocab.index_dataset(dataset, field_name='Sentence')\n", | |||||
"print(dataset)" | |||||
] | |||||
}, | |||||
{ | |||||
"cell_type": "markdown", | |||||
"id": "6b26b707", | |||||
"metadata": {}, | |||||
"source": [ | |||||
"最后,使用相同方法,再将`dataset`中`Sentiment`字段中的`negative`、`neutral`、`positive`转化为数字编号" | |||||
] | |||||
}, | |||||
{ | |||||
"cell_type": "code", | |||||
"execution_count": 29, | |||||
"id": "5f5eed18", | "id": "5f5eed18", | ||||
"metadata": {}, | "metadata": {}, | ||||
"outputs": [ | |||||
{ | |||||
"data": { | |||||
"text/html": [ | |||||
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n" | |||||
], | |||||
"text/plain": [] | |||||
}, | |||||
"metadata": {}, | |||||
"output_type": "display_data" | |||||
}, | |||||
{ | |||||
"name": "stdout", | |||||
"output_type": "stream", | |||||
"text": [ | |||||
"{'negative': 0, 'positive': 1, 'neutral': 2}\n" | |||||
] | |||||
}, | |||||
{ | |||||
"data": { | |||||
"text/html": [ | |||||
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n" | |||||
], | |||||
"text/plain": [] | |||||
}, | |||||
"metadata": {}, | |||||
"output_type": "display_data" | |||||
}, | |||||
{ | |||||
"data": { | |||||
"text/html": [ | |||||
"<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n", | |||||
"</pre>\n" | |||||
], | |||||
"text/plain": [ | |||||
"\n" | |||||
] | |||||
}, | |||||
"metadata": {}, | |||||
"output_type": "display_data" | |||||
}, | |||||
{ | |||||
"name": "stdout", | |||||
"output_type": "stream", | |||||
"text": [ | |||||
"+------------+------------------------------+-----------+\n", | |||||
"| SentenceId | Sentence | Sentiment |\n", | |||||
"+------------+------------------------------+-----------+\n", | |||||
"| 1 | [2, 14, 3, 15, 16, 5, 17,... | 0 |\n", | |||||
"| 2 | [12, 32, 4, 33, 8, 34, 35... | 1 |\n", | |||||
"| 3 | [38, 39, 3, 40, 41, 13, 4... | 0 |\n", | |||||
"| 4 | [2, 52, 53, 54, 3, 55, 8,... | 2 |\n", | |||||
"| 5 | [2, 67, 3, 68, 69, 70, 71... | 1 |\n", | |||||
"| 6 | [5, 81, 3, 82, 83, 4, 84,... | 2 |\n", | |||||
"+------------+------------------------------+-----------+\n" | |||||
] | |||||
} | |||||
], | |||||
"source": [ | |||||
"target_vocab = Vocabulary(padding=None, unknown=None)\n", | |||||
"\n", | |||||
"target_vocab.from_dataset(dataset, field_name='Sentiment')\n", | |||||
"print(target_vocab.word2idx)\n", | |||||
"target_vocab.index_dataset(dataset, field_name='Sentiment')\n", | |||||
"print(dataset)" | |||||
] | |||||
}, | |||||
{ | |||||
"cell_type": "markdown", | |||||
"id": "eed7ea64", | |||||
"metadata": {}, | |||||
"source": [ | |||||
"在最后的最后,通过以下的一张图,来总结本章关于`dataset`和`vocabulary`主要知识点的讲解,以及两者的联系\n", | |||||
"\n", | |||||
"<img src=\"./figures/T1-fig-dataset-and-vocabulary.png\" width=\"80%\" height=\"80%\" align=\"center\"></img>" | |||||
] | |||||
}, | |||||
{ | |||||
"cell_type": "code", | |||||
"execution_count": null, | |||||
"id": "35b4f0f7", | |||||
"metadata": {}, | |||||
"outputs": [], | "outputs": [], | ||||
"source": [] | "source": [] | ||||
} | } | ||||
@@ -0,0 +1,41 @@ | |||||
{ | |||||
"cells": [ | |||||
{ | |||||
"cell_type": "code", | |||||
"execution_count": null, | |||||
"metadata": { | |||||
"collapsed": true | |||||
}, | |||||
"outputs": [], | |||||
"source": [] | |||||
}, | |||||
{ | |||||
"cell_type": "code", | |||||
"execution_count": null, | |||||
"metadata": {}, | |||||
"outputs": [], | |||||
"source": [] | |||||
} | |||||
], | |||||
"metadata": { | |||||
"kernelspec": { | |||||
"display_name": "Python 3 (ipykernel)", | |||||
"language": "python", | |||||
"name": "python3" | |||||
}, | |||||
"language_info": { | |||||
"codemirror_mode": { | |||||
"name": "ipython", | |||||
"version": 3 | |||||
}, | |||||
"file_extension": ".py", | |||||
"mimetype": "text/x-python", | |||||
"name": "python", | |||||
"nbconvert_exporter": "python", | |||||
"pygments_lexer": "ipython3", | |||||
"version": "3.7.4" | |||||
} | |||||
}, | |||||
"nbformat": 4, | |||||
"nbformat_minor": 1 | |||||
} |