From 34698f0f9e9ecb43d9c7b4ec822615fa779cb0e5 Mon Sep 17 00:00:00 2001 From: x54-729 <17307130121@fudan.edu.cn> Date: Mon, 9 May 2022 11:34:15 +0000 Subject: [PATCH 01/15] =?UTF-8?q?=E5=88=A0=E9=99=A4=20mix=5Fmodule=20?= =?UTF-8?q?=E5=92=8C=20torch=5Fpaddle=5Fdriver=20=E7=9A=84=E5=86=85?= =?UTF-8?q?=E5=AE=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/core/__init__.py | 1 - fastNLP/core/drivers/__init__.py | 2 - .../drivers/torch_paddle_driver/__init__.py | 5 - .../torch_paddle_driver.py | 193 -------- .../core/drivers/torch_paddle_driver/utils.py | 4 - fastNLP/core/utils/__init__.py | 2 - fastNLP/core/utils/torch_paddle_utils.py | 49 -- fastNLP/modules/__init__.py | 9 - fastNLP/modules/mix_modules/__init__.py | 10 - fastNLP/modules/mix_modules/mix_module.py | 310 ------------- fastNLP/modules/mix_modules/utils.py | 233 ---------- .../drivers/torch_paddle_driver/__init__.py | 0 .../_test_torch_paddle_driver.py | 122 ----- .../torch_paddle_driver/_test_utils.py | 0 tests/core/utils/_test_torch_paddle_utils.py | 204 -------- tests/modules/__init__.py | 0 tests/modules/mix_modules/__init__.py | 0 tests/modules/mix_modules/_test_mix_module.py | 378 --------------- tests/modules/mix_modules/_test_utils.py | 435 ------------------ 19 files changed, 1957 deletions(-) delete mode 100644 fastNLP/core/drivers/torch_paddle_driver/__init__.py delete mode 100644 fastNLP/core/drivers/torch_paddle_driver/torch_paddle_driver.py delete mode 100644 fastNLP/core/drivers/torch_paddle_driver/utils.py delete mode 100644 fastNLP/core/utils/torch_paddle_utils.py delete mode 100644 fastNLP/modules/__init__.py delete mode 100644 fastNLP/modules/mix_modules/__init__.py delete mode 100644 fastNLP/modules/mix_modules/mix_module.py delete mode 100644 fastNLP/modules/mix_modules/utils.py delete mode 100644 tests/core/drivers/torch_paddle_driver/__init__.py delete mode 100644 tests/core/drivers/torch_paddle_driver/_test_torch_paddle_driver.py delete mode 100644 tests/core/drivers/torch_paddle_driver/_test_utils.py delete mode 100644 tests/core/utils/_test_torch_paddle_utils.py delete mode 100644 tests/modules/__init__.py delete mode 100644 tests/modules/mix_modules/__init__.py delete mode 100644 tests/modules/mix_modules/_test_mix_module.py delete mode 100644 tests/modules/mix_modules/_test_utils.py diff --git a/fastNLP/core/__init__.py b/fastNLP/core/__init__.py index 8800be8e..b0f71f52 100644 --- a/fastNLP/core/__init__.py +++ b/fastNLP/core/__init__.py @@ -63,7 +63,6 @@ __all__ = [ "PaddleFleetDriver", "JittorSingleDriver", "JittorMPIDriver", - "TorchPaddleDriver", # log "logger", diff --git a/fastNLP/core/drivers/__init__.py b/fastNLP/core/drivers/__init__.py index a67d886e..f9be3180 100644 --- a/fastNLP/core/drivers/__init__.py +++ b/fastNLP/core/drivers/__init__.py @@ -9,7 +9,6 @@ __all__ = [ "JittorDriver", "JittorSingleDriver", "JittorMPIDriver", - "TorchPaddleDriver", 'torch_seed_everything', 'paddle_seed_everything', 'optimizer_state_to_device' @@ -18,7 +17,6 @@ __all__ = [ from .torch_driver import TorchDriver, TorchSingleDriver, TorchDDPDriver, torch_seed_everything, optimizer_state_to_device from .jittor_driver import JittorDriver, JittorMPIDriver, JittorSingleDriver from .paddle_driver import PaddleDriver, PaddleFleetDriver, PaddleSingleDriver, paddle_seed_everything -from .torch_paddle_driver import TorchPaddleDriver from .driver import Driver diff --git a/fastNLP/core/drivers/torch_paddle_driver/__init__.py b/fastNLP/core/drivers/torch_paddle_driver/__init__.py deleted file mode 100644 index 6deeed73..00000000 --- a/fastNLP/core/drivers/torch_paddle_driver/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -__all__ = [ - "TorchPaddleDriver", -] - -from .torch_paddle_driver import TorchPaddleDriver \ No newline at end of file diff --git a/fastNLP/core/drivers/torch_paddle_driver/torch_paddle_driver.py b/fastNLP/core/drivers/torch_paddle_driver/torch_paddle_driver.py deleted file mode 100644 index 20be8a37..00000000 --- a/fastNLP/core/drivers/torch_paddle_driver/torch_paddle_driver.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Optional, Dict, Union, Callable, Tuple - -from fastNLP.envs.imports import _NEED_IMPORT_PADDLE, _NEED_IMPORT_TORCH -from fastNLP.core.utils.utils import _get_fun_msg - - -if _NEED_IMPORT_PADDLE: - import paddle - from paddle.io import DataLoader as PaddleDataLoader - from paddle.optimizer import Optimizer as PaddleOptimizer - -if _NEED_IMPORT_TORCH: - import torch - from torch.utils.data import DataLoader as TorchDataLoader - from torch.optim import Optimizer as TorchOptimizer - -from fastNLP.core.drivers.driver import Driver -from fastNLP.envs.distributed import rank_zero_call -from fastNLP.core.utils.utils import auto_param_call, apply_to_collection -from fastNLP.core.log.logger import logger -from fastNLP.modules.mix_modules.mix_module import MixModule - - -__all__ = [ - "TorchPaddleDriver", -] - -class TorchPaddleDriver(Driver): - """ - 针对torch和paddle混合模型的driver - 由于是两种不同的框架不方便实现多卡，暂时先实现CPU和GPU单卡的功能 - """ - def __init__(self, model, device: Optional[str] = None, **kwargs): - super(TorchPaddleDriver, self).__init__(model) - - self.model_device = device - self.torch_non_blocking = kwargs.get("torch_non_blocking", None) - self.paddle_blocking = kwargs.get("paddle_blocking", None) - - self._data_device = kwargs.get("_data_device", None) - if isinstance(self._data_device, int): - # 将data_device设置为cuda:x的字符串形式 - if self._data_device < 0: - raise ValueError("Parameter `_data_device` can not be smaller than 0.") - _could_use_device_num = paddle.device.cuda.device_count() - if self._data_device >= _could_use_device_num: - raise ValueError("The gpu device that parameter `device` specifies is not existed.") - self._data_device = f"cuda:{self._data_device}" - elif self._data_device is not None: - raise ValueError("Parameter `device` is wrong type, please check our documentation for the right use.") - - def setup(self): - if self.model_device is not None: - paddle.device.set_device(self.model_device.replace("cuda", "gpu")) - self.model.to(self.model_device) - - @staticmethod - def check_dataloader_legality(dataloader, dataloader_name, is_train: bool = False): - if is_train: - if not isinstance(dataloader, (TorchDataLoader, PaddleDataLoader)): - raise ValueError(f"Parameter `{dataloader_name}` should be 'torch.util.data.DataLoader' or `paddle.io.dataloader` type, not {type(dataloader)}.") - else: - if not isinstance(dataloader, Dict): - raise ValueError(f"Parameter `{dataloader_name}` should be 'Dict' type, not {type(dataloader)}.") - else: - for each_dataloader in dataloader.values(): - if not isinstance(each_dataloader, (TorchDataLoader, PaddleDataLoader)): - raise ValueError(f"Each dataloader of parameter `{dataloader_name}` should be " - f"'torch.util.data.DataLoader' or `paddle.io.dataloader` " - f"type, not {type(each_dataloader)}.") - - @staticmethod - def _check_optimizer_legality(optimizers): - for each_optimizer in optimizers: - if not isinstance(each_optimizer, (TorchOptimizer, PaddleOptimizer)): - raise ValueError(f"Each optimizers of parameter `optimizers` should be " - f"'torch.optim.Optimizer' or 'paddle.optimizers.Optimizer' type, " - f"not {type(each_optimizer)}.") - - def step(self): - for optimizer in self.optimizers: - optimizer.step() - - def backward(self, loss): - loss.backward() - - def zero_grad(self): - for optimizer in self.optimizers: - if isinstance(optimizer, TorchOptimizer): - optimizer.zero_grad() - elif isinstance(optimizer, PaddleOptimizer): - optimizer.clear_grad() - else: - raise ValueError("Unknown optimizers type.") - - def model_call(self, batch, fn: Callable, signature_fn: Optional[Callable]) -> Dict: - if isinstance(batch, Dict) and not self.wo_auto_param_call: - return auto_param_call(fn, batch, signature_fn=signature_fn) - else: - return fn(batch) - - def get_model_call_fn(self, fn: str) -> Tuple: - if hasattr(self.model, fn): - fn = getattr(self.model, fn) - if not callable(fn): - raise RuntimeError(f"The `{fn}` attribute is not `Callable`.") - logger.debug(f'Use {_get_fun_msg(fn, with_fp=False)}...') - return fn, None - elif fn in {"train_step", "evaluate_step"}: - logger.debug(f'Use {_get_fun_msg(self.model.forward, with_fp=False)}...') - return self.model, self.model.forward - else: - raise RuntimeError(f"There is no `{fn}` method in your {type(self.model)}.") - - def predict_step(self, batch): - if isinstance(batch, Dict): - return auto_param_call(self._predict_step, batch) - else: - return self._predict_step(batch) - - @rank_zero_call - def save_model(self, filepath: str, only_state_dict: bool = True, model_save_fn: Optional[Callable] = None): - r""" - 暂时不提供保存整个模型的方法 - """ - if only_state_dict == False: - logger.warn("TorchPaddleModule only support saving state dicts now.") - if model_save_fn is not None: - model_save_fn(filepath) - else: - model = self.unwrap_model() - self.move_model_to_device(model, "cpu") - self.model.save(filepath) - self.move_model_to_device(model, self.model_device) - - def load_model(self, filepath: str): - """ - 加载模型的加载函数； - - :param filepath: 保存文件的文件位置（需要包括文件名）； - :return: - """ - return self.model.load(filepath) - - def save(self): - ... - - def load(self): - ... - - @staticmethod - def move_model_to_device(model: MixModule, device: str): - if device is not None: - model.to(device) - - def unwrap_model(self): - return self.model - - @staticmethod - def tensor_to_numeric(tensor): - if tensor is None: - return None - - def _translate(_data): - return _data.tolist() - - return apply_to_collection( - data=tensor, - dtype=(paddle.Tensor, torch.Tensor), - function=_translate - ) - - def set_model_mode(self, mode: str): - assert mode in {"train", "eval"} - getattr(self.model, mode)() - - def get_model_device(self): - return self.model_device - - @property - def data_device(self): - if self.model_device is not None: - return self.model_device - else: - return self._data_device - - def set_model_mode(self, mode: str): - assert mode in {"train", "eval"} - getattr(self.model, mode)() - - def set_sampler_epoch(self, dataloader: Union['TorchDataLoader', 'PaddleDataLoader'], cur_epoch_idx): - # 保证 ddp 训练时的 shuffle=True 时的正确性，因为需要保证每一个进程上的 sampler 的shuffle 的随机数种子是一样的； - return dataloader diff --git a/fastNLP/core/drivers/torch_paddle_driver/utils.py b/fastNLP/core/drivers/torch_paddle_driver/utils.py deleted file mode 100644 index 328ac7ec..00000000 --- a/fastNLP/core/drivers/torch_paddle_driver/utils.py +++ /dev/null @@ -1,4 +0,0 @@ -from fastNLP.envs.imports import _NEED_IMPORT_PADDLE - -if _NEED_IMPORT_PADDLE: - pass \ No newline at end of file diff --git a/fastNLP/core/utils/__init__.py b/fastNLP/core/utils/__init__.py index 4de52d16..aca01344 100644 --- a/fastNLP/core/utils/__init__.py +++ b/fastNLP/core/utils/__init__.py @@ -11,7 +11,6 @@ __all__ = [ 'is_in_fnlp_paddle_dist', 'is_in_paddle_launch_dist', 'f_rich_progress', - 'torch_paddle_move_data_to_device', 'torch_move_data_to_device', 'get_fn_arg_names', 'auto_param_call', @@ -32,7 +31,6 @@ from .jittor_utils import is_jittor_dataset, jittor_collate_wraps from .paddle_utils import get_device_from_visible, paddle_to, paddle_move_data_to_device, get_paddle_device_id, get_paddle_gpu_str, is_in_paddle_dist, \ is_in_fnlp_paddle_dist, is_in_paddle_launch_dist from .rich_progress import f_rich_progress -from .torch_paddle_utils import torch_paddle_move_data_to_device from .torch_utils import torch_move_data_to_device from .utils import * diff --git a/fastNLP/core/utils/torch_paddle_utils.py b/fastNLP/core/utils/torch_paddle_utils.py deleted file mode 100644 index 9201548d..00000000 --- a/fastNLP/core/utils/torch_paddle_utils.py +++ /dev/null @@ -1,49 +0,0 @@ -from typing import Any, Optional - -from fastNLP.envs.imports import _NEED_IMPORT_PADDLE, _NEED_IMPORT_TORCH - -if _NEED_IMPORT_PADDLE: - import paddle - -if _NEED_IMPORT_TORCH: - import torch - -__all__ = [ - "torch_paddle_move_data_to_device", -] - -from .utils import apply_to_collection -from .paddle_utils import paddle_to - - -def torch_paddle_move_data_to_device(batch: Any, device: Optional[str] = None, non_blocking: Optional[bool] = True, - data_device: Optional[str] = None) -> Any: - - r""" - 将数据集合传输到给定设备。只有paddle.Tensor和torch.Tensor对象会被传输到设备中，其余保持不变 - - :param batch: - :param device: - :param non_blocking: - :param data_device: - :return: 相同的集合，但所有包含的张量都驻留在新设备上； - """ - - if device is None: - if data_device is not None: - device = data_device - else: - return batch - - torch_device = device.replace("gpu", "cuda") - paddle_device = device.replace("cuda", "gpu") - - def batch_to(data: Any) -> Any: - if isinstance(data, torch.Tensor): - data = data.to(torch_device, non_blocking=non_blocking) - elif isinstance(data, paddle.Tensor): - data = paddle_to(data, paddle_device) - - return data - - return apply_to_collection(batch, dtype=(paddle.Tensor, torch.Tensor), function=batch_to) \ No newline at end of file diff --git a/fastNLP/modules/__init__.py b/fastNLP/modules/__init__.py deleted file mode 100644 index a2da19c1..00000000 --- a/fastNLP/modules/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -__all__ = [ - "MixModule", - "torch2paddle", - "paddle2torch", - "torch2jittor", - "jittor2torch", -] - -from .mix_modules import MixModule, torch2paddle, paddle2torch, torch2jittor, jittor2torch \ No newline at end of file diff --git a/fastNLP/modules/mix_modules/__init__.py b/fastNLP/modules/mix_modules/__init__.py deleted file mode 100644 index 1e3b085d..00000000 --- a/fastNLP/modules/mix_modules/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -__all__ = [ - "MixModule", - "torch2paddle", - "paddle2torch", - "torch2jittor", - "jittor2torch", -] - -from .mix_module import MixModule -from .utils import * \ No newline at end of file diff --git a/fastNLP/modules/mix_modules/mix_module.py b/fastNLP/modules/mix_modules/mix_module.py deleted file mode 100644 index 40abcf51..00000000 --- a/fastNLP/modules/mix_modules/mix_module.py +++ /dev/null @@ -1,310 +0,0 @@ -import os -import io -import pickle -from typing import Dict -from collections import OrderedDict - -import numpy as np - -from fastNLP.envs.imports import _NEED_IMPORT_JITTOR, _NEED_IMPORT_PADDLE, _NEED_IMPORT_TORCH -from fastNLP.core.utils.paddle_utils import paddle_to - -if _NEED_IMPORT_PADDLE: - import paddle - from paddle.nn import Layer as PaddleLayer - -if _NEED_IMPORT_TORCH: - import torch - from torch.nn import Module as TorchModule, Parameter as TorchParameter - -if _NEED_IMPORT_JITTOR: - import jittor - - -__all__ = [ - "MixModule", -] - -class MixModule: - """ - TODO: 支持不同的混合方式；添加state_dict的支持；如果参数里有List of Tensors该怎么处理； - 是否需要仿照Module那样在初始化的时候给各种模型分类 - 可以同时使用Torch和Paddle框架的混合模型 - """ - def __init__(self, *args, **kwargs): - pass - - def __call__(self, *args, **kwargs): - return self.forward(*args, **kwargs) - - def named_parameters(self, prefix='', recurse: bool=True, backend=None): - """ - 返回模型的名字和参数 - - :param prefix: 输出时在参数名前加上的前缀 - :param recurse: 是否递归地输出参数 - :param backend: `backend`=`None`时，将所有模型和张量的参数返回； - `backend`=`torch`时，返回`torch`的参数； - `backend`=`paddle`时，返回`paddle`的参数。 - """ - if backend is None: - generator = self.attributes(TorchModule, TorchParameter, PaddleLayer) - elif backend == "torch": - generator = self.attributes(TorchModule, TorchParameter) - elif backend == "paddle": - generator = self.attributes(PaddleLayer) - else: - raise ValueError("Unknown backend parameter.") - - for name, value in generator: - name = prefix + ('.' if prefix else '') + name - if isinstance(value, TorchParameter): - # 非Module/Layer类型，直接输出名字和值 - yield name, value - elif recurse: - # 递归地调用named_parameters - for name_r, value_r in value.named_parameters(name, recurse): - yield name_r, value_r - - def parameters(self, recurse: bool = True, backend: str = None): - """ - 返回模型的参数 - - :param recurse: - :param backend: `backend`=`None`时，将所有模型和张量的参数返回； - `backend`=`torch`时，返回`torch`的参数； - `backend`=`paddle`时，返回`paddle`的参数。 - """ - for name, value in self.named_parameters(recurse=recurse, backend=backend): - yield value - - def forward(self, *args, **kwargs): - raise NotImplementedError - - def train_step(self, batch): - raise NotImplementedError - - def test_step(self, batch): - raise NotImplementedError - - def evaluate_step(self, batch): - raise NotImplementedError - - def train(self): - for name, value in self.attributes(TorchModule, PaddleLayer): - value.train() - - def eval(self): - for name, value in self.attributes(TorchModule, PaddleLayer): - value.eval() - - def to(self, device): - """ - :param device: 设备名 - """ - # 有jittor的话 warning - if device == "cpu": - paddle_device = device - elif device.startswith("cuda"): - paddle_device = device.replace("cuda", "gpu") - elif device.startswith("gpu"): - paddle_device = device - device = device.replace("gpu", "cuda") - else: - raise ValueError("Device value error") - - for name, value in self.attributes(TorchModule): - # torch的to函数不影响Tensor - vars(self)[name] = value.to(device) - for name, value in self.attributes(TorchParameter): - # Parameter在经过to函数后会变成Tensor类型 - vars(self)[name] = TorchParameter(value.to(device), requires_grad=value.requires_grad) - - for name, value in self.attributes(PaddleLayer): - vars(self)[name] = value.to(paddle_device) - for name, value in self.attributes(paddle.Tensor): - # paddle的to函数会影响到Tensor - vars(self)[name] = paddle_to(value, paddle_device) - - return self - - def state_dict(self, backend: str = None) -> Dict: - """ - 返回模型的state_dict。 - - .. note:: torch的destination参数会在将来删除，因此不提供destination参数 - - :param backend: `backend`=`None`时，将所有模型和张量的state dict返回； - `backend`=`torch`时，返回`torch`的state dict； - `backend`=`paddle`时，返回`paddle`的state dict。 - """ - if backend is None: - generator = self.attributes(TorchModule, TorchParameter, PaddleLayer) - elif backend == "torch": - generator = self.attributes(TorchModule, TorchParameter) - elif backend == "paddle": - generator = self.attributes(PaddleLayer) - else: - raise ValueError(f"Unknown backend {backend}.") - - destination = OrderedDict() - - for name, value in generator: - if value is None: - continue - if isinstance(value, TorchParameter): - destination[name] = value - else: - # 不同框架state_dict函数的参数名和顺序不同 - if isinstance(value, PaddleLayer): - kwargs = { - "structured_name_prefix": name + ".", - } - elif isinstance(value, TorchModule): - kwargs = { - "prefix": name + ".", - } - else: - raise ValueError(f"Unknown item type {type(value)}") - destination.update(value.state_dict(**kwargs)) - - return destination - - def save_state_dict_to_file(self, path: str): - """ - 保存模型的state dict到path - """ - # TODO 设备限制 - filename = os.path.basename(path) - if filename == "": - raise ValueError("Received empty filename.") - dirname = os.path.dirname(path) - if dirname and not os.path.exists(dirname): - os.makedirs(dirname) - protocol = 4 - - saved = {} - paddle_dict = self.state_dict(backend="paddle") - torch_dict = self.state_dict(backend="torch") - # 保存paddle部分 - # 调用paddle保存时的处理函数 - paddle_saved_obj = paddle.framework.io._build_saved_state_dict(paddle_dict) - paddle_saved_obj = paddle.fluid.io._unpack_saved_dict(paddle_saved_obj, protocol) - # 将返回的dict保存 - saved["paddle"] = paddle_saved_obj - - # 保存torch部分 - buffer = io.BytesIO() - torch.save(torch_dict, buffer) - saved["torch"] = buffer.getvalue() - - # 保存 - with open(path, "wb") as f: - pickle.dump(saved, f, protocol) - - def load_state_dict_from_file(self, path: str): - """ - 从 `path` 中加载保存的state dict - """ - state_dict = {} - with open(path, "rb") as f: - loaded = pickle.load(f) - # 加载paddle的数据 - paddle_loaded_obj = loaded["paddle"] - paddle_load_result = paddle.fluid.io._pack_loaded_dict(paddle_loaded_obj) - if "StructuredToParameterName@@" in paddle_load_result: - for key in paddle_load_result["StructuredToParameterName@@"]: - if isinstance(paddle_load_result[key], np.ndarray): - paddle_load_result[key] = paddle.to_tensor(paddle_load_result[key]) - state_dict.update(paddle_load_result) - # 加载torch的数据 - torch_loaded_obj = loaded["torch"] - torch_bytes = io.BytesIO(torch_loaded_obj) - torch_load_result = torch.load(torch_bytes) - state_dict.update(torch_load_result) - - self.load_state_dict(state_dict) - - def load_state_dict(self, state_dict): - """ - 从state dict中加载数据 - """ - missing_keys = [] - unexpected_keys = [] - error_msgs = [] - new_state = {} - - local_state = self.state_dict() - - # 对字典内容按前缀进行归类 - for key, value in state_dict.items(): - splited = key.split(".", 1) - if len(splited) == 1: - # 没有前缀，实际上只有torch.nn.Parameter会进入这种情况 - new_state[key] = value - else: - prefix, name = splited - if prefix not in new_state: - new_state[prefix] = {} - new_state[prefix][name] = value - - for key, param in self.attributes(TorchModule, TorchParameter, PaddleLayer): - if key in new_state: - # 在传入的字典中找到了对应的值 - input_param = new_state[key] - if not isinstance(input_param, dict): - # 且不是字典，即上述没有前缀的情况 - # 按照torch.nn.Module._load_from_state_dict进行赋值 - if not torch.overrides.is_tensor_like(input_param): - error_msgs.append('While copying the parameter named "{}", ' - 'expected torch.Tensor or Tensor-like object from checkpoint but ' - 'received {}' - .format(key, type(input_param))) - continue - - # This is used to avoid copying uninitialized parameters into - # non-lazy modules, since they dont have the hook to do the checks - # in such case, it will error when accessing the .shape attribute. - is_param_lazy = torch.nn.parameter.is_lazy(param) - # Backward compatibility: loading 1-dim tensor from 0.3.* to version 0.4+ - if not is_param_lazy and len(param.shape) == 0 and len(input_param.shape) == 1: - input_param = input_param[0] - - if not is_param_lazy and input_param.shape != param.shape: - # local shape should match the one in checkpoint - error_msgs.append('size mismatch for {}: copying a param with shape {} from checkpoint, ' - 'the shape in current model is {}.' - .format(key, input_param.shape, param.shape)) - continue - try: - with torch.no_grad(): - param.copy_(input_param) - except Exception as ex: - error_msgs.append('While copying the parameter named "{}", ' - 'whose dimensions in the model are {} and ' - 'whose dimensions in the checkpoint are {}, ' - 'an exception occurred : {}.' - .format(key, param.size(), input_param.size(), ex.args)) - else: - # 否则在子模块中 - if isinstance(param, TorchModule): - # torch模块 - # 由于paddle没有提供类似strict的参数，因此也不对torch作要求 - param.load_state_dict(input_param, strict=False) - elif isinstance(param, PaddleLayer): - # paddle模块 - param.load_dict(input_param) - else: - missing_keys.append(key) - - if len(error_msgs) > 0: - raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format( - self.__class__.__name__, "\n\t".join(error_msgs))) - - def attributes(self, *types): - """ - 查找对应类型的成员 - """ - for name, value in vars(self).items(): - if isinstance(value, types): - yield name, value diff --git a/fastNLP/modules/mix_modules/utils.py b/fastNLP/modules/mix_modules/utils.py deleted file mode 100644 index 5d56ffee..00000000 --- a/fastNLP/modules/mix_modules/utils.py +++ /dev/null @@ -1,233 +0,0 @@ -import warnings -import os -from typing import Any, Optional, Union - -import numpy as np - -from fastNLP.core.utils.utils import apply_to_collection -from fastNLP.core.utils.paddle_utils import paddle_to -from fastNLP.envs.imports import _NEED_IMPORT_JITTOR, _NEED_IMPORT_TORCH, _NEED_IMPORT_PADDLE - -if _NEED_IMPORT_PADDLE: - import paddle - -if _NEED_IMPORT_JITTOR: - import jittor - -if _NEED_IMPORT_TORCH: - import torch - -__all__ = [ - "paddle2torch", - "torch2paddle", - "jittor2torch", - "torch2jittor", -] - -def _paddle2torch(paddle_tensor: 'paddle.Tensor', target_device: Optional[Union[str, int]] = None, no_gradient: bool = None) -> 'torch.Tensor': - """ - 将paddle tensor转换为torch tensor，并且能够保留梯度进行反向传播 - :param paddle_tensor: 要转换的paddle张量 - :param target_device: 是否将转换后的张量迁移到特定设备上，输入为`None`时，和输入的张量相同。 - :param no_gradient: 是否保留原张量的梯度。为`None`时，新的张量与输入张量保持一致； - 为`True`时，全部不保留梯度；为`False`时，全部保留梯度。 - :return: 转换后的torch张量 - """ - no_gradient = paddle_tensor.stop_gradient if no_gradient is None else no_gradient - paddle_numpy = paddle_tensor.numpy() - if not np.issubdtype(paddle_numpy.dtype, np.inexact): - no_gradient = True - - if target_device is None: - if paddle_tensor.place.is_gpu_place(): - # paddlepaddle有两种Place，对应不同的device id获取方式 - if hasattr(paddle_tensor.place, "gpu_device_id"): - # paddle.fluid.core_avx.Place - # 在gpu环境下创建张量的话，张量的place是这一类型 - target_device = f"cuda:{paddle_tensor.place.gpu_device_id()}" - else: - # paddle.CUDAPlace - target_device = f"cuda:{paddle_tensor.place.get_device_id()}" - else: - # TODO: 可能需要支持xpu等设备 - target_device = "cpu" - - if not no_gradient: - # 保持梯度，并保持反向传播 - # torch.tensor会保留numpy数组的类型 - torch_tensor = torch.tensor(paddle_numpy, requires_grad=True, device=target_device) - hook = torch_tensor.register_hook( - lambda grad: paddle.autograd.backward(paddle_tensor, paddle.to_tensor(grad.cpu().numpy())) - ) - else: - # 不保留梯度 - torch_tensor = torch.tensor(paddle_numpy, requires_grad=False, device=target_device) - - return torch_tensor - - -def _torch2paddle(torch_tensor: 'torch.Tensor', target_device: str = None, no_gradient: bool = None) -> 'paddle.Tensor': - """ - 将torch tensor转换为paddle tensor，并且能够保留梯度进行反向传播。 - :param torch_tensor: 要转换的torch张量 - :param target_device: 是否将转换后的张量迁移到特定设备上，输入为`None`时，和输入的张量相同。 - :param no_gradient: 是否保留原张量的梯度。为`None`时，新的张量与输入张量保持一致； - 为`True`时，全部不保留梯度；为`False`时，全部保留梯度。 - :return: 转换后的paddle张量 - """ - no_gradient = not torch_tensor.requires_grad if no_gradient is None else no_gradient - if target_device is None: - if torch_tensor.is_cuda: - target_device = f"gpu:{torch_tensor.device.index}" - else: - target_device = "cpu" - - if not no_gradient: - # 保持梯度并保持反向传播 - # paddle的stop_gradient和torch的requires_grad表现是相反的 - paddle_tensor = paddle.to_tensor(torch_tensor.detach().numpy(), stop_gradient=False) - hook = paddle_tensor.register_hook( - lambda grad: torch.autograd.backward(torch_tensor, torch.tensor(grad.numpy())) - ) - else: - paddle_tensor = paddle.to_tensor(torch_tensor.detach().numpy(), stop_gradient=True) - - paddle_tensor = paddle_to(paddle_tensor, target_device) - - return paddle_tensor - - -def _jittor2torch(jittor_var: 'jittor.Var', target_device: Optional[Union[str, int]] = None, no_gradient: bool = None) -> 'torch.Tensor': - """ - 将jittor Var转换为torch tensor，并且能够保留梯度进行反向传播 - :param jittor_var: 要转换的jittor变量 - :param target_device: 是否将转换后的张量迁移到特定设备上，输入为`None`时，根据jittor.flags.use_cuda决定。 - :param no_gradient: 是否保留原张量的梯度。为`None`时，新的张量与输入张量保持一致； - 为`True`时，全部不保留梯度；为`False`时，全部保留梯度。 - :return: 转换后的torch张量 - """ - # TODO: warning：无法保留梯度 - # jittor的grad可以通过callback进行传递 - # 如果outputs有_grad键，可以实现求导 - no_gradient = not jittor_var.requires_grad if no_gradient is None else no_gradient - if no_gradient == False: - warnings.warn("The result tensor will not keep gradients due to differences between jittor and pytorch.") - jittor_numpy = jittor_var.numpy() - if not np.issubdtype(jittor_numpy.dtype, np.inexact): - no_gradient = True - - if target_device is None: - # jittor的设备分配是自动的 - # 根据use_cuda判断 - if jittor.flags.use_cuda: - target_device = "cuda:0" - else: - target_device = "cpu" - - torch_tensor = torch.tensor(jittor_numpy, requires_grad=not no_gradient, device=target_device) - - return torch_tensor - - -def _torch2jittor(torch_tensor: 'torch.Tensor', no_gradient: bool = None) -> 'jittor.Var': - """ - 将torch tensor转换为jittor Var，并且能够保留梯度进行反向传播 - :param torch_tensor: 要转换的torch张量 - :param no_gradient: 是否保留原张量的梯度。为`None`时，新的张量与输入张量保持一致； - 为`True`时，全部不保留梯度；为`False`时，全部保留梯度。 - :return: 转换后的jittor变量 - """ - no_gradient = not torch_tensor.requires_grad if no_gradient is None else no_gradient - - if not no_gradient: - # 保持梯度并保持反向传播 - jittor_var = jittor.Var(torch_tensor.detach().numpy()) - jittor_var.requires_grad = True - hook = jittor_var.register_hook( - lambda grad: torch.autograd.backward(torch_tensor, torch.tensor(grad.numpy())) - ) - else: - jittor_var = jittor.Var(torch_tensor.detach().numpy()) - jittor_var.requires_grad = False - - return jittor_var - - -def torch2paddle(torch_in: Any, target_device: str = None, no_gradient: bool = None) -> Any: - """ - 递归地将输入中包含的torch张量转换为paddle张量 - - :param torch_in: 要转换的包含torch.Tensor类型的变量 - :param target_device: 是否将转换后的张量迁移到特定设备上， - 输入为`None`时，和输入的张量相同， - :param no_gradient: 是否保留原张量的梯度。为`None`时，新的张量与输入张量保持一致； - 为`True`时，全部不保留梯度；为`False`时，全部保留梯度。 - :return: 将所有torch.Tensor转换为paddle.Tensor的张量 - """ - - return apply_to_collection( - torch_in, - dtype=torch.Tensor, - function=_torch2paddle, - target_device=target_device, - no_gradient=no_gradient, - ) - - -def paddle2torch(paddle_in: Any, target_device: str = None, no_gradient: bool = None) -> Any: - """ - 递归地将输入中包含的paddle张量转换为torch张量 - - :param torch_in: 要转换的包含paddle.Tensor类型的变量 - :param target_device: 是否将转换后的张量迁移到特定设备上， - 输入为`None`时，和输入的张量相同， - :param no_gradient: 是否保留原张量的梯度。为`None`时，新的张量与输入张量保持一致； - 为`True`时，全部不保留梯度；为`False`时，全部保留梯度。 - :return: 将所有paddle.Tensor转换为torch.Tensor后的变量 - """ - - return apply_to_collection( - paddle_in, - dtype=paddle.Tensor, - function=_paddle2torch, - target_device=target_device, - no_gradient=no_gradient, - ) - - -def jittor2torch(jittor_in: Any, target_device: str = None, no_gradient: bool = None) -> Any: - """ - 递归地将输入中包含的jittor变量转换为torch张量 - - :param jittor_in: 要转换的jittor变量 - :param target_device: 是否将转换后的张量迁移到特定设备上，输入为`None`时，默认为cuda:0。 - :param no_gradient: 是否保留原张量的梯度。为`None`时，新的张量与输入张量保持一致； - 为`True`时，全部不保留梯度；为`False`时，全部保留梯度。 - :return: 转换后的torch张量 - """ - - return apply_to_collection( - jittor_in, - dtype=jittor.Var, - function=_jittor2torch, - target_device=target_device, - no_gradient=no_gradient, - ) - - -def torch2jittor(torch_in: Any, no_gradient: bool = None) -> Any: - """ - 递归地将输入中包含的torch张量转换为jittor变量 - - :param torch_tensor: 要转换的torch张量 - :param no_gradient: 是否保留原张量的梯度。为`None`时，新的张量与输入张量保持一致； - 为`True`时，全部不保留梯度；为`False`时，全部保留梯度。 - :return: 转换后的jittor变量 - """ - - return apply_to_collection( - torch_in, - dtype=torch.Tensor, - function=_torch2jittor, - no_gradient=no_gradient, - ) \ No newline at end of file diff --git a/tests/core/drivers/torch_paddle_driver/__init__.py b/tests/core/drivers/torch_paddle_driver/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/core/drivers/torch_paddle_driver/_test_torch_paddle_driver.py b/tests/core/drivers/torch_paddle_driver/_test_torch_paddle_driver.py deleted file mode 100644 index 76b19ba4..00000000 --- a/tests/core/drivers/torch_paddle_driver/_test_torch_paddle_driver.py +++ /dev/null @@ -1,122 +0,0 @@ -import pytest - -from fastNLP.modules.mix_modules.mix_module import MixModule -from fastNLP.core.drivers.torch_paddle_driver.torch_paddle_driver import TorchPaddleDriver -from fastNLP.modules.mix_modules.utils import paddle2torch, torch2paddle - -import torch -import paddle -from paddle.io import Dataset, DataLoader -import numpy as np - -############################################################################ -# -# 测试在MNIST数据集上的表现 -# -############################################################################ - -class MNISTDataset(Dataset): - def __init__(self, dataset): - - self.dataset = [ - ( - np.array(img).astype('float32').reshape(-1), - label - ) for img, label in dataset - ] - - def __getitem__(self, idx): - return self.dataset[idx] - - def __len__(self): - return len(self.dataset) - -class MixMNISTModel(MixModule): - def __init__(self): - super(MixMNISTModel, self).__init__() - - self.fc1 = paddle.nn.Linear(784, 64) - self.fc2 = paddle.nn.Linear(64, 32) - self.fc3 = torch.nn.Linear(32, 10) - self.fc4 = torch.nn.Linear(10, 10) - - def forward(self, x): - - paddle_out = self.fc1(x) - paddle_out = self.fc2(paddle_out) - torch_in = paddle2torch(paddle_out) - torch_out = self.fc3(torch_in) - torch_out = self.fc4(torch_out) - - return torch_out - - def train_step(self, x): - return self.forward(x) - - def test_step(self, x): - return self.forward(x) - -@pytest.mark.torchpaddle -class TestMNIST: - - @classmethod - def setup_class(self): - - self.train_dataset = paddle.vision.datasets.MNIST(mode='train') - self.test_dataset = paddle.vision.datasets.MNIST(mode='test') - self.train_dataset = MNISTDataset(self.train_dataset) - - self.lr = 0.0003 - self.epochs = 20 - - self.dataloader = DataLoader(self.train_dataset, batch_size=100, shuffle=True) - - def setup_method(self): - - model = MixMNISTModel() - self.torch_loss_func = torch.nn.CrossEntropyLoss() - - torch_opt = torch.optim.Adam(model.parameters(backend="torch"), self.lr) - paddle_opt = paddle.optimizer.Adam(parameters=model.parameters(backend="paddle"), learning_rate=self.lr) - - self.driver = TorchPaddleDriver(model=model, device="cuda:0") - self.driver.set_optimizers([torch_opt, paddle_opt]) - - def test_case1(self): - - epochs = 20 - - self.driver.setup() - self.driver.zero_grad() - # 开始训练 - current_epoch_idx = 0 - while current_epoch_idx < epochs: - epoch_loss, batch = 0, 0 - self.driver.set_model_mode("train") - self.driver.set_sampler_epoch(self.dataloader, current_epoch_idx) - for batch, (img, label) in enumerate(self.dataloader): - img = paddle.to_tensor(img).cuda() - torch_out = self.driver.train_step(img) - label = torch.from_numpy(label.numpy()).reshape(-1) - loss = self.torch_loss_func(torch_out.cpu(), label) - epoch_loss += loss.item() - - self.driver.backward(loss) - self.driver.step() - self.driver.zero_grad() - - current_epoch_idx += 1 - - # 开始测试 - correct = 0 - for img, label in self.test_dataset: - - img = paddle.to_tensor(np.array(img).astype('float32').reshape(1, -1)) - torch_out = self.driver.test_step(img) - res = torch_out.softmax(-1).argmax().item() - label = label.item() - if res == label: - correct += 1 - - acc = correct / len(self.test_dataset) - assert acc > 0.85 diff --git a/tests/core/drivers/torch_paddle_driver/_test_utils.py b/tests/core/drivers/torch_paddle_driver/_test_utils.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/core/utils/_test_torch_paddle_utils.py b/tests/core/utils/_test_torch_paddle_utils.py deleted file mode 100644 index e10b1d11..00000000 --- a/tests/core/utils/_test_torch_paddle_utils.py +++ /dev/null @@ -1,204 +0,0 @@ -import paddle -import pytest -import torch - -from fastNLP.core.utils.torch_paddle_utils import torch_paddle_move_data_to_device - -############################################################################ -# -# 测试将参数中包含的所有torch和paddle张量迁移到指定设备 -# -############################################################################ - -@pytest.mark.torchpaddle -class TestTorchPaddleMoveDataToDevice: - - def check_gpu(self, tensor, idx): - """ - 检查张量是否在指定显卡上的工具函数 - """ - - if isinstance(tensor, paddle.Tensor): - assert tensor.place.is_gpu_place() - assert tensor.place.gpu_device_id() == idx - elif isinstance(tensor, torch.Tensor): - assert tensor.is_cuda - assert tensor.device.index == idx - - def check_cpu(self, tensor): - if isinstance(tensor, paddle.Tensor): - assert tensor.place.is_cpu_place() - elif isinstance(tensor, torch.Tensor): - assert not tensor.is_cuda - - def test_tensor_transfer(self): - """ - 测试迁移单个张量 - """ - - paddle_tensor = paddle.rand((3, 4, 5)).cpu() - res = torch_paddle_move_data_to_device(paddle_tensor, device=None, data_device=None) - self.check_cpu(res) - - res = torch_paddle_move_data_to_device(paddle_tensor, device="gpu:0", data_device=None) - self.check_gpu(res, 0) - - res = torch_paddle_move_data_to_device(paddle_tensor, device="gpu:1", data_device=None) - self.check_gpu(res, 1) - - res = torch_paddle_move_data_to_device(paddle_tensor, device="cuda:0", data_device="cpu") - self.check_gpu(res, 0) - - res = torch_paddle_move_data_to_device(paddle_tensor, device=None, data_device="gpu:0") - self.check_gpu(res, 0) - - res = torch_paddle_move_data_to_device(paddle_tensor, device=None, data_device="cuda:1") - self.check_gpu(res, 1) - - torch_tensor = torch.rand(3, 4, 5) - res = torch_paddle_move_data_to_device(torch_tensor, device=None, data_device=None) - self.check_cpu(res) - - res = torch_paddle_move_data_to_device(torch_tensor, device="gpu:0", data_device=None) - self.check_gpu(res, 0) - - res = torch_paddle_move_data_to_device(torch_tensor, device="gpu:1", data_device=None) - self.check_gpu(res, 1) - - res = torch_paddle_move_data_to_device(torch_tensor, device="gpu:0", data_device="cpu") - self.check_gpu(res, 0) - - res = torch_paddle_move_data_to_device(torch_tensor, device=None, data_device="gpu:0") - self.check_gpu(res, 0) - - res = torch_paddle_move_data_to_device(torch_tensor, device=None, data_device="gpu:1") - self.check_gpu(res, 1) - - def test_list_transfer(self): - """ - 测试迁移张量的列表 - """ - - paddle_list = [paddle.rand((6, 4, 2)) for i in range(5)] + [torch.rand((6, 4, 2)) for i in range(5)] - res = torch_paddle_move_data_to_device(paddle_list, device=None, data_device="gpu:1") - assert isinstance(res, list) - for r in res: - self.check_gpu(r, 1) - - res = torch_paddle_move_data_to_device(paddle_list, device="cpu", data_device="gpu:1") - assert isinstance(res, list) - for r in res: - self.check_cpu(r) - - res = torch_paddle_move_data_to_device(paddle_list, device="gpu:0", data_device=None) - assert isinstance(res, list) - for r in res: - self.check_gpu(r, 0) - - res = torch_paddle_move_data_to_device(paddle_list, device="gpu:1", data_device="cpu") - assert isinstance(res, list) - for r in res: - self.check_gpu(r, 1) - - def test_tensor_tuple_transfer(self): - """ - 测试迁移张量的元组 - """ - - paddle_list = [paddle.rand((6, 4, 2)) for i in range(10)] + [torch.rand((6, 4, 2)) for i in range(5)] - paddle_tuple = tuple(paddle_list) - res = torch_paddle_move_data_to_device(paddle_tuple, device=None, data_device="gpu:1") - assert isinstance(res, tuple) - for r in res: - self.check_gpu(r, 1) - - res = torch_paddle_move_data_to_device(paddle_tuple, device="cpu", data_device="gpu:1") - assert isinstance(res, tuple) - for r in res: - self.check_cpu(r) - - res = torch_paddle_move_data_to_device(paddle_tuple, device="gpu:0", data_device=None) - assert isinstance(res, tuple) - for r in res: - self.check_gpu(r, 0) - - res = torch_paddle_move_data_to_device(paddle_tuple, device="gpu:1", data_device="cpu") - assert isinstance(res, tuple) - for r in res: - self.check_gpu(r, 1) - - def test_dict_transfer(self): - """ - 测试迁移复杂的字典结构 - """ - - paddle_dict = { - "torch_tensor": torch.rand((3, 4)), - "torch_list": [torch.rand((6, 4, 2)) for i in range(10)], - "dict":{ - "list": [paddle.rand((6, 4, 2)) for i in range(5)] + [torch.rand((6, 4, 2)) for i in range(5)], - "torch_tensor": torch.rand((3, 4)), - "paddle_tensor": paddle.rand((3, 4)) - }, - "paddle_tensor": paddle.rand((3, 4)), - "list": [paddle.rand((6, 4, 2)) for i in range(10)] , - "int": 2, - "string": "test string" - } - - res = torch_paddle_move_data_to_device(paddle_dict, device="gpu:0", data_device=None) - assert isinstance(res, dict) - self.check_gpu(res["torch_tensor"], 0) - self.check_gpu(res["paddle_tensor"], 0) - assert isinstance(res["torch_list"], list) - for t in res["torch_list"]: - self.check_gpu(t, 0) - assert isinstance(res["list"], list) - for t in res["list"]: - self.check_gpu(t, 0) - assert isinstance(res["int"], int) - assert isinstance(res["string"], str) - assert isinstance(res["dict"], dict) - assert isinstance(res["dict"]["list"], list) - for t in res["dict"]["list"]: - self.check_gpu(t, 0) - self.check_gpu(res["dict"]["torch_tensor"], 0) - self.check_gpu(res["dict"]["paddle_tensor"], 0) - - res = torch_paddle_move_data_to_device(paddle_dict, device=None, data_device="gpu:1") - assert isinstance(res, dict) - self.check_gpu(res["torch_tensor"], 1) - self.check_gpu(res["paddle_tensor"], 1) - assert isinstance(res["torch_list"], list) - for t in res["torch_list"]: - self.check_gpu(t, 1) - assert isinstance(res["list"], list) - for t in res["list"]: - self.check_gpu(t, 1) - assert isinstance(res["int"], int) - assert isinstance(res["string"], str) - assert isinstance(res["dict"], dict) - assert isinstance(res["dict"]["list"], list) - for t in res["dict"]["list"]: - self.check_gpu(t, 1) - self.check_gpu(res["dict"]["torch_tensor"], 1) - self.check_gpu(res["dict"]["paddle_tensor"], 1) - - res = torch_paddle_move_data_to_device(paddle_dict, device="cpu", data_device="gpu:0") - assert isinstance(res, dict) - self.check_cpu(res["torch_tensor"]) - self.check_cpu(res["paddle_tensor"]) - assert isinstance(res["torch_list"], list) - for t in res["torch_list"]: - self.check_cpu(t) - assert isinstance(res["list"], list) - for t in res["list"]: - self.check_cpu(t) - assert isinstance(res["int"], int) - assert isinstance(res["string"], str) - assert isinstance(res["dict"], dict) - assert isinstance(res["dict"]["list"], list) - for t in res["dict"]["list"]: - self.check_cpu(t) - self.check_cpu(res["dict"]["torch_tensor"]) - self.check_cpu(res["dict"]["paddle_tensor"]) diff --git a/tests/modules/__init__.py b/tests/modules/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/modules/mix_modules/__init__.py b/tests/modules/mix_modules/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/modules/mix_modules/_test_mix_module.py b/tests/modules/mix_modules/_test_mix_module.py deleted file mode 100644 index 87206fd6..00000000 --- a/tests/modules/mix_modules/_test_mix_module.py +++ /dev/null @@ -1,378 +0,0 @@ -import pytest -import os -from itertools import chain - -import torch -import paddle -from paddle.io import Dataset, DataLoader -import numpy as np - -from fastNLP.modules.mix_modules.mix_module import MixModule -from fastNLP.modules.mix_modules.utils import paddle2torch, torch2paddle -from fastNLP.envs.distributed import rank_zero_rm - - -############################################################################ -# -# 测试类的基本功能 -# -############################################################################ - -class MixModuleForTest(MixModule): - def __init__(self): - super(MixModuleForTest, self).__init__() - - self.torch_fc1 = torch.nn.Linear(10, 10) - self.torch_softmax = torch.nn.Softmax(0) - self.torch_conv2d1 = torch.nn.Conv2d(10, 10, 3) - self.torch_tensor = torch.ones(3, 3) - self.torch_param = torch.nn.Parameter(torch.ones(4, 4)) - - self.paddle_fc1 = paddle.nn.Linear(10, 10) - self.paddle_softmax = paddle.nn.Softmax(0) - self.paddle_conv2d1 = paddle.nn.Conv2D(10, 10, 3) - self.paddle_tensor = paddle.ones((4, 4)) - -class TorchModuleForTest(torch.nn.Module): - def __init__(self): - super(TorchModuleForTest, self).__init__() - - self.torch_fc1 = torch.nn.Linear(10, 10) - self.torch_softmax = torch.nn.Softmax(0) - self.torch_conv2d1 = torch.nn.Conv2d(10, 10, 3) - self.torch_tensor = torch.ones(3, 3) - self.torch_param = torch.nn.Parameter(torch.ones(4, 4)) - -class PaddleModuleForTest(paddle.nn.Layer): - def __init__(self): - super(PaddleModuleForTest, self).__init__() - - self.paddle_fc1 = paddle.nn.Linear(10, 10) - self.paddle_softmax = paddle.nn.Softmax(0) - self.paddle_conv2d1 = paddle.nn.Conv2D(10, 10, 3) - self.paddle_tensor = paddle.ones((4, 4)) - - -@pytest.mark.torchpaddle -class TestTorchPaddleMixModule: - - def setup_method(self): - - self.model = MixModuleForTest() - self.torch_model = TorchModuleForTest() - self.paddle_model = PaddleModuleForTest() - - def test_to(self): - """ - 测试混合模型的to函数 - """ - - self.model.to("cuda") - self.torch_model.to("cuda") - self.paddle_model.to("gpu") - self.if_device_correct("cuda") - - self.model.to("cuda:2") - self.torch_model.to("cuda:2") - self.paddle_model.to("gpu:2") - self.if_device_correct("cuda:2") - - self.model.to("gpu:1") - self.torch_model.to("cuda:1") - self.paddle_model.to("gpu:1") - self.if_device_correct("cuda:1") - - self.model.to("cpu") - self.torch_model.to("cpu") - self.paddle_model.to("cpu") - self.if_device_correct("cpu") - - def test_train_eval(self): - """ - 测试train和eval函数 - """ - - self.model.eval() - self.if_training_correct(False) - - self.model.train() - self.if_training_correct(True) - - def test_parameters(self): - """ - 测试parameters()函数，由于初始化是随机的，目前仅比较得到结果的长度 - """ - mix_params = [] - params = [] - - for value in self.model.named_parameters(): - mix_params.append(value) - - for value in chain(self.torch_model.named_parameters(), self.paddle_model.named_parameters()): - params.append(value) - - assert len(params) == len(mix_params) - - def test_named_parameters(self): - """ - 测试named_parameters函数 - """ - - mix_param_names = [] - param_names = [] - - for name, value in self.model.named_parameters(): - mix_param_names.append(name) - - for name, value in chain(self.torch_model.named_parameters(), self.paddle_model.named_parameters()): - param_names.append(name) - - assert sorted(param_names) == sorted(mix_param_names) - - def test_torch_named_parameters(self): - """ - 测试对torch参数的提取 - """ - - mix_param_names = [] - param_names = [] - - for name, value in self.model.named_parameters(backend="torch"): - mix_param_names.append(name) - - for name, value in self.torch_model.named_parameters(): - param_names.append(name) - - assert sorted(param_names) == sorted(mix_param_names) - - def test_paddle_named_parameters(self): - """ - 测试对paddle参数的提取 - """ - - mix_param_names = [] - param_names = [] - - for name, value in self.model.named_parameters(backend="paddle"): - mix_param_names.append(name) - - for name, value in self.paddle_model.named_parameters(): - param_names.append(name) - - assert sorted(param_names) == sorted(mix_param_names) - - def test_torch_state_dict(self): - """ - 测试提取torch的state dict - """ - torch_dict = self.torch_model.state_dict() - mix_dict = self.model.state_dict(backend="torch") - - assert sorted(torch_dict.keys()) == sorted(mix_dict.keys()) - - def test_paddle_state_dict(self): - """ - 测试提取paddle的state dict - """ - paddle_dict = self.paddle_model.state_dict() - mix_dict = self.model.state_dict(backend="paddle") - - # TODO 测试程序会显示passed后显示paddle的异常退出信息 - assert sorted(paddle_dict.keys()) == sorted(mix_dict.keys()) - - def test_state_dict(self): - """ - 测试提取所有的state dict - """ - all_dict = self.torch_model.state_dict() - all_dict.update(self.paddle_model.state_dict()) - mix_dict = self.model.state_dict() - - # TODO 测试程序会显示passed后显示paddle的异常退出信息 - assert sorted(all_dict.keys()) == sorted(mix_dict.keys()) - - def test_load_state_dict(self): - """ - 测试load_state_dict函数 - """ - state_dict = self.model.state_dict() - - new_model = MixModuleForTest() - new_model.load_state_dict(state_dict) - new_state_dict = new_model.state_dict() - - for name, value in state_dict.items(): - state_dict[name] = value.tolist() - for name, value in new_state_dict.items(): - new_state_dict[name] = value.tolist() - - # self.assertDictEqual(state_dict, new_state_dict) - - def test_save_and_load_state_dict(self): - """ - 测试save_state_dict_to_file和load_state_dict_from_file函数 - """ - path = "model" - try: - self.model.save_state_dict_to_file(path) - new_model = MixModuleForTest() - new_model.load_state_dict_from_file(path) - - state_dict = self.model.state_dict() - new_state_dict = new_model.state_dict() - - for name, value in state_dict.items(): - state_dict[name] = value.tolist() - for name, value in new_state_dict.items(): - new_state_dict[name] = value.tolist() - - # self.assertDictEqual(state_dict, new_state_dict) - finally: - rank_zero_rm(path) - - def if_device_correct(self, device): - - - assert self.model.torch_fc1.weight.device == self.torch_model.torch_fc1.weight.device - assert self.model.torch_conv2d1.weight.device == self.torch_model.torch_fc1.bias.device - assert self.model.torch_conv2d1.bias.device == self.torch_model.torch_conv2d1.bias.device - assert self.model.torch_tensor.device == self.torch_model.torch_tensor.device - assert self.model.torch_param.device == self.torch_model.torch_param.device - - if device == "cpu": - assert self.model.paddle_fc1.weight.place.is_cpu_place() - assert self.model.paddle_fc1.bias.place.is_cpu_place() - assert self.model.paddle_conv2d1.weight.place.is_cpu_place() - assert self.model.paddle_conv2d1.bias.place.is_cpu_place() - assert self.model.paddle_tensor.place.is_cpu_place() - elif device.startswith("cuda"): - assert self.model.paddle_fc1.weight.place.is_gpu_place() - assert self.model.paddle_fc1.bias.place.is_gpu_place() - assert self.model.paddle_conv2d1.weight.place.is_gpu_place() - assert self.model.paddle_conv2d1.bias.place.is_gpu_place() - assert self.model.paddle_tensor.place.is_gpu_place() - - assert self.model.paddle_fc1.weight.place.gpu_device_id() == self.paddle_model.paddle_fc1.weight.place.gpu_device_id() - assert self.model.paddle_fc1.bias.place.gpu_device_id() == self.paddle_model.paddle_fc1.bias.place.gpu_device_id() - assert self.model.paddle_conv2d1.weight.place.gpu_device_id() == self.paddle_model.paddle_conv2d1.weight.place.gpu_device_id() - assert self.model.paddle_conv2d1.bias.place.gpu_device_id() == self.paddle_model.paddle_conv2d1.bias.place.gpu_device_id() - assert self.model.paddle_tensor.place.gpu_device_id() == self.paddle_model.paddle_tensor.place.gpu_device_id() - else: - raise NotImplementedError - - def if_training_correct(self, training): - - assert self.model.torch_fc1.training == training - assert self.model.torch_softmax.training == training - assert self.model.torch_conv2d1.training == training - - assert self.model.paddle_fc1.training == training - assert self.model.paddle_softmax.training == training - assert self.model.paddle_conv2d1.training == training - - -############################################################################ -# -# 测试在MNIST数据集上的表现 -# -############################################################################ - -class MNISTDataset(Dataset): - def __init__(self, dataset): - - self.dataset = [ - ( - np.array(img).astype('float32').reshape(-1), - label - ) for img, label in dataset - ] - - def __getitem__(self, idx): - return self.dataset[idx] - - def __len__(self): - return len(self.dataset) - -class MixMNISTModel(MixModule): - def __init__(self): - super(MixMNISTModel, self).__init__() - - self.fc1 = paddle.nn.Linear(784, 64) - self.fc2 = paddle.nn.Linear(64, 32) - self.fc3 = torch.nn.Linear(32, 10) - self.fc4 = torch.nn.Linear(10, 10) - - def forward(self, x): - - paddle_out = self.fc1(x) - paddle_out = self.fc2(paddle_out) - torch_in = paddle2torch(paddle_out) - torch_out = self.fc3(torch_in) - torch_out = self.fc4(torch_out) - - return torch_out - -@pytest.mark.torchpaddle -class TestMNIST: - - @classmethod - def setup_class(self): - - self.train_dataset = paddle.vision.datasets.MNIST(mode='train') - self.test_dataset = paddle.vision.datasets.MNIST(mode='test') - self.train_dataset = MNISTDataset(self.train_dataset) - - self.lr = 0.0003 - self.epochs = 20 - - self.dataloader = DataLoader(self.train_dataset, batch_size=100, shuffle=True) - - def setup_method(self): - - self.model = MixMNISTModel().to("cuda") - self.torch_loss_func = torch.nn.CrossEntropyLoss() - - self.torch_opt = torch.optim.Adam(self.model.parameters(backend="torch"), self.lr) - self.paddle_opt = paddle.optimizer.Adam(parameters=self.model.parameters(backend="paddle"), learning_rate=self.lr) - - def test_case1(self): - - # 开始训练 - for epoch in range(self.epochs): - epoch_loss, batch = 0, 0 - for batch, (img, label) in enumerate(self.dataloader): - - img = paddle.to_tensor(img).cuda() - torch_out = self.model(img) - label = torch.from_numpy(label.numpy()).reshape(-1) - loss = self.torch_loss_func(torch_out.cpu(), label) - epoch_loss += loss.item() - - loss.backward() - self.torch_opt.step() - self.paddle_opt.step() - self.torch_opt.zero_grad() - self.paddle_opt.clear_grad() - - else: - assert epoch_loss / (batch + 1) < 0.3 - - # 开始测试 - correct = 0 - for img, label in self.test_dataset: - - img = paddle.to_tensor(np.array(img).astype('float32').reshape(1, -1)) - torch_out = self.model(img) - res = torch_out.softmax(-1).argmax().item() - label = label.item() - if res == label: - correct += 1 - - acc = correct / len(self.test_dataset) - assert acc > 0.85 - -############################################################################ -# -# 测试在ERNIE中文数据集上的表现 -# -############################################################################ diff --git a/tests/modules/mix_modules/_test_utils.py b/tests/modules/mix_modules/_test_utils.py deleted file mode 100644 index ea7e55d7..00000000 --- a/tests/modules/mix_modules/_test_utils.py +++ /dev/null @@ -1,435 +0,0 @@ -import unittest -import os - -os.environ["log_silent"] = "1" -import torch -import paddle -import jittor - -from fastNLP.modules.mix_modules.utils import ( - paddle2torch, - torch2paddle, - jittor2torch, - torch2jittor, -) - -############################################################################ -# -# 测试paddle到torch的转换 -# -############################################################################ - -class Paddle2TorchTestCase(unittest.TestCase): - - def check_torch_tensor(self, tensor, device, requires_grad): - """ - 检查张量设备和梯度情况的工具函数 - """ - - assert isinstance(tensor, torch.Tensor) - assert tensor.device == torch.device(device) - assert tensor.requires_grad == requires_grad - - def test_gradient(self): - """ - 测试张量转换后的反向传播是否正确 - """ - - x = paddle.to_tensor([1.0, 2.0, 3.0, 4.0, 5.0], stop_gradient=False) - y = paddle2torch(x) - z = 3 * (y ** 2) - z.sum().backward() - assert y.grad.tolist() == [6, 12, 18, 24, 30] - - def test_tensor_transfer(self): - """ - 测试单个张量的设备和梯度转换是否正确 - """ - - paddle_tensor = paddle.rand((3, 4, 5)).cpu() - res = paddle2torch(paddle_tensor) - self.check_torch_tensor(res, "cpu", not paddle_tensor.stop_gradient) - - res = paddle2torch(paddle_tensor, target_device="cuda:2", no_gradient=None) - self.check_torch_tensor(res, "cuda:2", not paddle_tensor.stop_gradient) - - res = paddle2torch(paddle_tensor, target_device="cuda:1", no_gradient=True) - self.check_torch_tensor(res, "cuda:1", False) - - res = paddle2torch(paddle_tensor, target_device="cuda:1", no_gradient=False) - self.check_torch_tensor(res, "cuda:1", True) - - def test_list_transfer(self): - """ - 测试张量列表的转换 - """ - - paddle_list = [paddle.rand((6, 4, 2)).cuda(1) for i in range(10)] - res = paddle2torch(paddle_list) - assert isinstance(res, list) - for t in res: - self.check_torch_tensor(t, "cuda:1", False) - - res = paddle2torch(paddle_list, target_device="cpu", no_gradient=False) - assert isinstance(res, list) - for t in res: - self.check_torch_tensor(t, "cpu", True) - - def test_tensor_tuple_transfer(self): - """ - 测试张量元组的转换 - """ - - paddle_list = [paddle.rand((6, 4, 2)).cuda(1) for i in range(10)] - paddle_tuple = tuple(paddle_list) - res = paddle2torch(paddle_tuple) - assert isinstance(res, tuple) - for t in res: - self.check_torch_tensor(t, "cuda:1", False) - - def test_dict_transfer(self): - """ - 测试包含复杂结构的字典的转换 - """ - - paddle_dict = { - "tensor": paddle.rand((3, 4)).cuda(0), - "list": [paddle.rand((6, 4, 2)).cuda(0) for i in range(10)], - "dict":{ - "list": [paddle.rand((6, 4, 2)).cuda(0) for i in range(10)], - "tensor": paddle.rand((3, 4)).cuda(0) - }, - "int": 2, - "string": "test string" - } - res = paddle2torch(paddle_dict) - assert isinstance(res, dict) - self.check_torch_tensor(res["tensor"], "cuda:0", False) - assert isinstance(res["list"], list) - for t in res["list"]: - self.check_torch_tensor(t, "cuda:0", False) - assert isinstance(res["int"], int) - assert isinstance(res["string"], str) - assert isinstance(res["dict"], dict) - assert isinstance(res["dict"]["list"], list) - for t in res["dict"]["list"]: - self.check_torch_tensor(t, "cuda:0", False) - self.check_torch_tensor(res["dict"]["tensor"], "cuda:0", False) - - -############################################################################ -# -# 测试torch到paddle的转换 -# -############################################################################ - -class Torch2PaddleTestCase(unittest.TestCase): - - def check_paddle_tensor(self, tensor, device, stop_gradient): - """ - 检查得到的paddle张量设备和梯度情况的工具函数 - """ - - assert isinstance(tensor, paddle.Tensor) - if device == "cpu": - assert tensor.place.is_cpu_place() - elif device.startswith("gpu"): - paddle_device = paddle.device._convert_to_place(device) - assert tensor.place.is_gpu_place() - if hasattr(tensor.place, "gpu_device_id"): - # paddle中，有两种Place - # paddle.fluid.core.Place是创建Tensor时使用的类型 - # 有函数gpu_device_id获取设备 - assert tensor.place.gpu_device_id() == paddle_device.get_device_id() - else: - # 通过_convert_to_place得到的是paddle.CUDAPlace - # 通过get_device_id获取设备 - assert tensor.place.get_device_id() == paddle_device.get_device_id() - else: - raise NotImplementedError - assert tensor.stop_gradient == stop_gradient - - def test_gradient(self): - """ - 测试转换后梯度的反向传播 - """ - - x = torch.tensor([1.0, 2.0, 3.0, 4.0, 5.0], requires_grad=True) - y = torch2paddle(x) - z = 3 * (y ** 2) - z.sum().backward() - assert y.grad.tolist() == [6, 12, 18, 24, 30] - - def test_tensor_transfer(self): - """ - 测试单个张量的转换 - """ - - torch_tensor = torch.rand((3, 4, 5)) - res = torch2paddle(torch_tensor) - self.check_paddle_tensor(res, "cpu", True) - - res = torch2paddle(torch_tensor, target_device="gpu:2", no_gradient=None) - self.check_paddle_tensor(res, "gpu:2", True) - - res = torch2paddle(torch_tensor, target_device="gpu:2", no_gradient=True) - self.check_paddle_tensor(res, "gpu:2", True) - - res = torch2paddle(torch_tensor, target_device="gpu:2", no_gradient=False) - self.check_paddle_tensor(res, "gpu:2", False) - - def test_tensor_list_transfer(self): - """ - 测试张量列表的转换 - """ - - torch_list = [torch.rand(6, 4, 2) for i in range(10)] - res = torch2paddle(torch_list) - assert isinstance(res, list) - for t in res: - self.check_paddle_tensor(t, "cpu", True) - - res = torch2paddle(torch_list, target_device="gpu:1", no_gradient=False) - assert isinstance(res, list) - for t in res: - self.check_paddle_tensor(t, "gpu:1", False) - - def test_tensor_tuple_transfer(self): - """ - 测试张量元组的转换 - """ - - torch_list = [torch.rand(6, 4, 2) for i in range(10)] - torch_tuple = tuple(torch_list) - res = torch2paddle(torch_tuple, target_device="cpu") - assert isinstance(res, tuple) - for t in res: - self.check_paddle_tensor(t, "cpu", True) - - def test_dict_transfer(self): - """ - 测试复杂的字典结构的转换 - """ - - torch_dict = { - "tensor": torch.rand((3, 4)), - "list": [torch.rand(6, 4, 2) for i in range(10)], - "dict":{ - "list": [torch.rand(6, 4, 2) for i in range(10)], - "tensor": torch.rand((3, 4)) - }, - "int": 2, - "string": "test string" - } - res = torch2paddle(torch_dict) - assert isinstance(res, dict) - self.check_paddle_tensor(res["tensor"], "cpu", True) - assert isinstance(res["list"], list) - for t in res["list"]: - self.check_paddle_tensor(t, "cpu", True) - assert isinstance(res["int"], int) - assert isinstance(res["string"], str) - assert isinstance(res["dict"], dict) - assert isinstance(res["dict"]["list"], list) - for t in res["dict"]["list"]: - self.check_paddle_tensor(t, "cpu", True) - self.check_paddle_tensor(res["dict"]["tensor"], "cpu", True) - - -############################################################################ -# -# 测试jittor到torch的转换 -# -############################################################################ - -class Jittor2TorchTestCase(unittest.TestCase): - - def check_torch_tensor(self, tensor, device, requires_grad): - """ - 检查得到的torch张量的工具函数 - """ - - assert isinstance(tensor, torch.Tensor) - if device == "cpu": - assert not tensor.is_cuda - else: - assert tensor.device == torch.device(device) - assert tensor.requires_grad == requires_grad - - def test_var_transfer(self): - """ - 测试单个Jittor Var的转换 - """ - - jittor_var = jittor.rand((3, 4, 5)) - res = jittor2torch(jittor_var) - self.check_torch_tensor(res, "cpu", True) - - res = jittor2torch(jittor_var, target_device="cuda:2", no_gradient=None) - self.check_torch_tensor(res, "cuda:2", True) - - res = jittor2torch(jittor_var, target_device="cuda:2", no_gradient=True) - self.check_torch_tensor(res, "cuda:2", False) - - res = jittor2torch(jittor_var, target_device="cuda:2", no_gradient=False) - self.check_torch_tensor(res, "cuda:2", True) - - def test_var_list_transfer(self): - """ - 测试Jittor列表的转换 - """ - - jittor_list = [jittor.rand((6, 4, 2)) for i in range(10)] - res = jittor2torch(jittor_list) - assert isinstance(res, list) - for t in res: - self.check_torch_tensor(t, "cpu", True) - - res = jittor2torch(jittor_list, target_device="cuda:1", no_gradient=False) - assert isinstance(res, list) - for t in res: - self.check_torch_tensor(t, "cuda:1", True) - - def test_var_tuple_transfer(self): - """ - 测试Jittor变量元组的转换 - """ - - jittor_list = [jittor.rand((6, 4, 2)) for i in range(10)] - jittor_tuple = tuple(jittor_list) - res = jittor2torch(jittor_tuple, target_device="cpu") - assert isinstance(res, tuple) - for t in res: - self.check_torch_tensor(t, "cpu", True) - - def test_dict_transfer(self): - """ - 测试字典结构的转换 - """ - - jittor_dict = { - "tensor": jittor.rand((3, 4)), - "list": [jittor.rand(6, 4, 2) for i in range(10)], - "dict":{ - "list": [jittor.rand(6, 4, 2) for i in range(10)], - "tensor": jittor.rand((3, 4)) - }, - "int": 2, - "string": "test string" - } - res = jittor2torch(jittor_dict) - assert isinstance(res, dict) - self.check_torch_tensor(res["tensor"], "cpu", True) - assert isinstance(res["list"], list) - for t in res["list"]: - self.check_torch_tensor(t, "cpu", True) - assert isinstance(res["int"], int) - assert isinstance(res["string"], str) - assert isinstance(res["dict"], dict) - assert isinstance(res["dict"]["list"], list) - for t in res["dict"]["list"]: - self.check_torch_tensor(t, "cpu", True) - self.check_torch_tensor(res["dict"]["tensor"], "cpu", True) - - -############################################################################ -# -# 测试torch到jittor的转换 -# -############################################################################ - -class Torch2JittorTestCase(unittest.TestCase): - - def check_jittor_var(self, var, requires_grad): - """ - 检查得到的Jittor Var梯度情况的工具函数 - """ - - assert isinstance(var, jittor.Var) - assert var.requires_grad == requires_grad - - def test_gradient(self): - """ - 测试反向传播的梯度 - """ - - x = torch.tensor([1.0, 2.0, 3.0, 4.0, 5.0], requires_grad=True) - y = torch2jittor(x) - z = 3 * (y ** 2) - grad = jittor.grad(z, y) - assert grad.tolist() == [6.0, 12.0, 18.0, 24.0, 30.0] - - def test_tensor_transfer(self): - """ - 测试单个张量转换为Jittor - """ - - torch_tensor = torch.rand((3, 4, 5)) - res = torch2jittor(torch_tensor) - self.check_jittor_var(res, False) - - res = torch2jittor(torch_tensor, no_gradient=None) - self.check_jittor_var(res, False) - - res = torch2jittor(torch_tensor, no_gradient=True) - self.check_jittor_var(res, False) - - res = torch2jittor(torch_tensor, no_gradient=False) - self.check_jittor_var(res, True) - - def test_tensor_list_transfer(self): - """ - 测试张量列表的转换 - """ - - torch_list = [torch.rand((6, 4, 2)) for i in range(10)] - res = torch2jittor(torch_list) - assert isinstance(res, list) - for t in res: - self.check_jittor_var(t, False) - - res = torch2jittor(torch_list, no_gradient=False) - assert isinstance(res, list) - for t in res: - self.check_jittor_var(t, True) - - def test_tensor_tuple_transfer(self): - """ - 测试张量元组的转换 - """ - - torch_list = [torch.rand((6, 4, 2)) for i in range(10)] - torch_tuple = tuple(torch_list) - res = torch2jittor(torch_tuple) - assert isinstance(res, tuple) - for t in res: - self.check_jittor_var(t, False) - - def test_dict_transfer(self): - """ - 测试字典结构的转换 - """ - - torch_dict = { - "tensor": torch.rand((3, 4)), - "list": [torch.rand(6, 4, 2) for i in range(10)], - "dict":{ - "list": [torch.rand(6, 4, 2) for i in range(10)], - "tensor": torch.rand((3, 4)) - }, - "int": 2, - "string": "test string" - } - res = torch2jittor(torch_dict) - assert isinstance(res, dict) - self.check_jittor_var(res["tensor"], False) - assert isinstance(res["list"], list) - for t in res["list"]: - self.check_jittor_var(t, False) - assert isinstance(res["int"], int) - assert isinstance(res["string"], str) - assert isinstance(res["dict"], dict) - assert isinstance(res["dict"]["list"], list) - for t in res["dict"]["list"]: - self.check_jittor_var(t, False) - self.check_jittor_var(res["dict"]["tensor"], False) From 2a44af25190d82fd3a46cd867b2cd2701e09f012 Mon Sep 17 00:00:00 2001 From: x54-729 <17307130121@fudan.edu.cn> Date: Mon, 9 May 2022 11:34:55 +0000 Subject: [PATCH 02/15] =?UTF-8?q?=E4=B8=BA=20test=5Ftrainer=5Fjittor=20?= =?UTF-8?q?=E6=B7=BB=E5=8A=A0=20DummyClass=20=E5=92=8C=20pytest.mark.jitto?= =?UTF-8?q?r?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/core/controllers/test_trainer_jittor.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/core/controllers/test_trainer_jittor.py b/tests/core/controllers/test_trainer_jittor.py index d0eac8cd..30e5e668 100644 --- a/tests/core/controllers/test_trainer_jittor.py +++ b/tests/core/controllers/test_trainer_jittor.py @@ -11,6 +11,9 @@ if _NEED_IMPORT_JITTOR: import jittor as jt from jittor import nn, Module from jittor.dataset import Dataset +else: + from fastNLP.core.utils.dummy_class import DummyClass as Module + from fastNLP.core.utils.dummy_class import DummyClass as Dataset class JittorNormalModel_Classification(Module): @@ -68,6 +71,7 @@ class TrainJittorConfig: @pytest.mark.parametrize("driver,device", [("jittor", None)]) @pytest.mark.parametrize("callbacks", [[RichCallback(100)]]) +@pytest.mark.jittor def test_trainer_jittor( driver, device, From 13fd67a041a1a3e4ab67fb48a0694e46c134eaa2 Mon Sep 17 00:00:00 2001 From: x54-729 <17307130121@fudan.edu.cn> Date: Mon, 9 May 2022 12:54:08 +0000 Subject: [PATCH 03/15] =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=B7=B7=E5=90=88?= =?UTF-8?q?=E6=A8=A1=E5=9E=8B=E7=9B=B8=E5=85=B3=E7=9A=84=E8=AE=BE=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Makefile b/docs/Makefile index a3710195..9f807ae2 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -6,7 +6,7 @@ SPHINXOPTS = SPHINXAPIDOC = sphinx-apidoc SPHINXBUILD = sphinx-build SPHINXPROJ = fastNLP -SPHINXEXCLUDE = ../fastNLP/transformers/* ../fastNLP/modules/* ../fastNLP/core/drivers/torch_paddle_driver/* ../fastNLP/core/utils/torch_paddle_utils.py +SPHINXEXCLUDE = ../fastNLP/transformers/* SOURCEDIR = source BUILDDIR = build PORT = 9000 From cb5aa370158994abfb7edb63c464ef93b9fe70f6 Mon Sep 17 00:00:00 2001 From: yh_cc Date: Mon, 9 May 2022 21:52:43 +0800 Subject: [PATCH 04/15] =?UTF-8?q?=E8=A1=A5=E5=85=85=E9=83=A8=E5=88=86?= =?UTF-8?q?=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/core/callbacks/callback_manager.py | 6 +++-- fastNLP/core/callbacks/checkpoint_callback.py | 27 +++++-------------- .../callbacks/load_best_model_callback.py | 3 ++- fastNLP/core/callbacks/topk_saver.py | 5 ++-- fastNLP/core/collators/padders/get_padder.py | 2 +- .../core/collators/padders/paddle_padder.py | 5 +++- .../core/collators/padders/torch_padder.py | 4 +-- fastNLP/core/controllers/evaluator.py | 8 ++++-- fastNLP/core/controllers/trainer.py | 2 +- .../core/dataloaders/prepare_dataloader.py | 2 +- fastNLP/core/dataset/dataset.py | 4 +-- .../torch_driver/initialize_torch_driver.py | 4 +-- .../core/drivers/torch_driver/torch_driver.py | 6 +++++ tests/core/collators/test_collator.py | 4 +-- 14 files changed, 42 insertions(+), 40 deletions(-) diff --git a/fastNLP/core/callbacks/callback_manager.py b/fastNLP/core/callbacks/callback_manager.py index eabc489b..f34c5dd3 100644 --- a/fastNLP/core/callbacks/callback_manager.py +++ b/fastNLP/core/callbacks/callback_manager.py @@ -10,8 +10,8 @@ from .callback_event import Event from .callback import Callback from fastNLP.core.log import logger from .progress_callback import ProgressCallback, choose_progress_callback -from fastNLP.envs import rank_zero_call -from fastNLP.core.utils.utils import _get_fun_msg +from ..utils.exceptions import EarlyStopException +from ..utils.utils import _get_fun_msg def _transfer(func): @@ -25,6 +25,8 @@ def _transfer(func): for callback_fn in manager.callback_fns[func.__name__]: try: callback_fn(*arg, **kwargs) + except EarlyStopException as e: + raise e except BaseException as e: logger.error(f"The following callback_fn raise exception:{_get_fun_msg(callback_fn)}.") raise e diff --git a/fastNLP/core/callbacks/checkpoint_callback.py b/fastNLP/core/callbacks/checkpoint_callback.py index a51406af..c4ed8d47 100644 --- a/fastNLP/core/callbacks/checkpoint_callback.py +++ b/fastNLP/core/callbacks/checkpoint_callback.py @@ -19,7 +19,7 @@ class CheckpointCallback(Callback): only_state_dict: bool = True, model_save_fn: Optional[Callable] = None, save_object: str = 'model', save_evaluate_results=True, **kwargs): """ - 保存模型 checkpoint 的 callback ，其保存的文件目录以及文件名命名规则如下:: + 保存 checkpoint 的 callback ，其保存的文件目录以及文件名命名规则如下:: - folder/ - YYYY-mm-dd-HH_MM_SS_fffff/ # 自动根据当前脚本的启动时间创建的 @@ -29,8 +29,9 @@ class CheckpointCallback(Callback): - {save_object}-epoch_{epoch_idx}-batch_{global_batch_idx}-exception_{exception_type}/ # exception时保存。 - {save_object}-epoch_{epoch_idx}-batch_{global_batch_idx}-{monitor}_{monitor_value}/ # 满足topk条件存储文件名 - model_save_fn 为 None ，则以上每个 folder 中，将生成 fastnlp_model.pkl.tar 文件。 - 若 model_save_fn 不为 None，则 fastNLP 将 folder 绝对路径传递给该函数，fastNLP 在该 folder 下不进行模型保存。 + model_save_fn 为 None ，则以上每个 folder 中，将生成 fastnlp_model.pkl.tar 文件。若 model_save_fn 不为 None， + 则 fastNLP 将 folder 绝对路径传递给该函数，fastNLP 在该 folder 下不进行模型保存。默认情况下，本 checkpoint 只保存了 model + 的状态；如还需保存 Trainer 的状态以断点重训的话，请使用 ``save_object='trainer'`` 。 :param monitor: 监控的 metric 值。如果在 evaluation 结果中没有找到完全一致的名称，将使用 最长公共字符串算法 找到最匹配 的那个作为 monitor 。如果为 None，将尝试使用 Trainer 设置的 monitor 。也可以传入一个函数，接受参数为 evaluation 的结 @@ -46,22 +47,14 @@ class CheckpointCallback(Callback): :param only_state_dict: 保存模型时是否只保存 state_dict 。当 model_save_fn 不为 None 时，该参数无效。 :param model_save_fn: 个性化的保存函数，当触发保存操作时，就调用这个函数，这个函数应当接受一个文件夹作为参数，不返回任何东西。 如果传入了 model_save_fn 函数，fastNLP 将不再进行模型相关的保存。在多卡场景下，我们只在 rank 0 上会运行该函数。 - :param save_object: 可选 ['trainer', 'model']，表示在保存时的保存对象为 trainer+model 还是 只是model 。 + :param save_object: 可选 ['trainer', 'model']，表示在保存时的保存对象为 ``trainer+model`` 还是 只是 ``model`` 。如果 + 保存 ``trainer`` 对象的话，将会保存 :class:~fastNLP.Trainer 的相关状态，可以通过 :meth:`Trainer.load` 加载该断 + 点继续训练。如果保存的是 ``Model`` 对象，则可以通过 :meth:`Trainer.load_model` 加载该模型权重。 :param save_evaluate_results: 是否保存 evaluate 的结果。如果为 True ，在保存 topk 模型的 folder 中还将额外保存一个 fastnlp_evaluate_results.json 文件，记录当前的 results。仅在设置了 topk 的场景下有用，默认为 True 。 :param kwargs: """ super().__init__() - if folder is None: - logger.warning( - "Parameter `folder` is None, and we will use the current work directory to find and load your model.") - folder = Path.cwd() - folder = Path(folder) - if not folder.exists(): - raise NotADirectoryError(f"Path '{folder.absolute()}' is not existed!") - elif folder.is_file(): - raise ValueError("Parameter `folder` should be a directory instead of a file.") - if every_n_epochs is not None: if not isinstance(every_n_epochs, int) or every_n_epochs < 1: raise ValueError("Parameter `every_n_epochs` should be an int and greater than or equal to 1.") @@ -74,12 +67,6 @@ class CheckpointCallback(Callback): else: every_n_batches = sys.maxsize # 使得没有数字可以整除 - if topk is not None: - if not isinstance(topk, int): - raise ValueError("Parameter `topk` should be an int.") - else: - topk = 0 - if on_exceptions is not None: if not isinstance(on_exceptions, Sequence): on_exceptions = [on_exceptions] diff --git a/fastNLP/core/callbacks/load_best_model_callback.py b/fastNLP/core/callbacks/load_best_model_callback.py index 362716ef..55ef40ad 100644 --- a/fastNLP/core/callbacks/load_best_model_callback.py +++ b/fastNLP/core/callbacks/load_best_model_callback.py @@ -19,7 +19,8 @@ class LoadBestModelCallback(HasMonitorCallback): model_load_fn:Optional[Callable] = None, delete_after_train:bool = True): """ - 保存最佳的 monitor 值最佳的模型，并在训练结束的时候重新加载模型。仅在训练正常结束的时候才能加载最好的模型。 + 保存最佳的 monitor 值最佳的模型，并在训练结束的时候重新加载模型，默认会在加载之后删除权重文件。仅在训练正常结束的时候才能加载 + 最好的模型。 :param str monitor: 监控的 metric 值。如果在 evaluation 结果中没有找到完全一致的名称，将使用 最长公共字符串算法 找到最匹配 的那个作为 monitor 。如果为 None，将尝试使用 Trainer 设置的 monitor 。也可以传入一个函数，接受参数为 evaluation 的结 diff --git a/fastNLP/core/callbacks/topk_saver.py b/fastNLP/core/callbacks/topk_saver.py index aba2ff63..c629e9de 100644 --- a/fastNLP/core/callbacks/topk_saver.py +++ b/fastNLP/core/callbacks/topk_saver.py @@ -33,9 +33,8 @@ class Saver: :param kwargs: 更多需要传递给 Trainer.save() 或者 Trainer.save_model() 接口的参数。 """ if folder is None: - logger.rank_zero_warning( - "Parameter `folder` is None, and we will use the current work directory to find and load your model.") - folder = Path.cwd() + folder = Path.cwd().absolute() + logger.info(f"Parameter `folder` is None, and we will use {folder} to save and load your model.") folder = Path(folder) if not folder.exists(): folder.mkdir(parents=True, exist_ok=True) diff --git a/fastNLP/core/collators/padders/get_padder.py b/fastNLP/core/collators/padders/get_padder.py index db48011b..e76391aa 100644 --- a/fastNLP/core/collators/padders/get_padder.py +++ b/fastNLP/core/collators/padders/get_padder.py @@ -121,7 +121,7 @@ def get_padder(batch_field:Sequence[Any], pad_val, dtype, backend, field_name)-> # 这里 ele_dtype 传入为 None 的原因是防止出现 paddle tensor 转换为 torch tensor return TorchTensorPadder(pad_val=pad_val, ele_dtype=None, dtype=dtype) elif backend == 'paddle': - return PaddleTensorPadder(pad_val=pad_val, ele_dtype=None, dtype=dtype) + return PaddleTensorPadder(pad_val=pad_val, ele_dtype=ele_dtype, dtype=dtype) elif backend == 'jittor': return JittorTensorPadder(pad_val=pad_val, ele_dtype=ele_dtype, dtype=dtype) else: diff --git a/fastNLP/core/collators/padders/paddle_padder.py b/fastNLP/core/collators/padders/paddle_padder.py index c4dbdadc..826d21c7 100644 --- a/fastNLP/core/collators/padders/paddle_padder.py +++ b/fastNLP/core/collators/padders/paddle_padder.py @@ -141,7 +141,10 @@ class PaddleTensorPadder(Padder): shapes = [field.shape for field in batch_field] max_shape = [len(batch_field)] + [max(*_) for _ in zip(*shapes)] - array = np.full(max_shape, fill_value=pad_val) + if isinstance(batch_field[0], paddle.Tensor): + array = paddle.full(max_shape, fill_value=pad_val, dtype=dtype) + else: + array = np.full(max_shape, fill_value=pad_val, dtype=batch_field[0].dtype) for i, field in enumerate(batch_field): slices = (i, ) + tuple(slice(0, s) for s in shapes[i]) array[slices] = field diff --git a/fastNLP/core/collators/padders/torch_padder.py b/fastNLP/core/collators/padders/torch_padder.py index b67aeff8..aaa0d4e9 100644 --- a/fastNLP/core/collators/padders/torch_padder.py +++ b/fastNLP/core/collators/padders/torch_padder.py @@ -118,8 +118,8 @@ class TorchTensorPadder(Padder): batch_field = [torch.tensor(field.tolist(), dtype=dtype) for field in batch_field] else: device = batch_field[0].device - if dtype is None: - dtype = batch_field[0].dtype + if dtype is None: + dtype = batch_field[0].dtype except AttributeError: raise RuntimeError(f"If the field is not a torch.Tensor (it is {type(batch_field[0])}), " f"it must have tolist() method.") diff --git a/fastNLP/core/controllers/evaluator.py b/fastNLP/core/controllers/evaluator.py index abd70644..03adf102 100644 --- a/fastNLP/core/controllers/evaluator.py +++ b/fastNLP/core/controllers/evaluator.py @@ -234,8 +234,7 @@ class Evaluator: """ 调用所有 metric 的 reset() 方法，清除累积的状态。 - Returns: - + :return: """ self.metrics_wrapper.reset() @@ -357,6 +356,11 @@ class _MetricsWrapper: metric.update(res) def reset(self): + """ + 将 Metric 中的状态重新设置。 + + :return: + """ for metric in self._metrics: if _is_allennlp_metric(metric): metric.get_metric(reset=True) diff --git a/fastNLP/core/controllers/trainer.py b/fastNLP/core/controllers/trainer.py index 9de400ab..9673fc78 100644 --- a/fastNLP/core/controllers/trainer.py +++ b/fastNLP/core/controllers/trainer.py @@ -646,7 +646,7 @@ class Trainer(TrainerEventTrigger): self.driver.save_model(folder, only_state_dict, **kwargs) self.driver.barrier() - def load_model(self, folder: Union[str, Path, BinaryIO, io.BytesIO], only_state_dict: bool = False, + def load_model(self, folder: Union[str, Path, BinaryIO, io.BytesIO], only_state_dict: bool = True, model_load_fn: Optional[Callable] = None, **kwargs): """ 加载模型 diff --git a/fastNLP/core/dataloaders/prepare_dataloader.py b/fastNLP/core/dataloaders/prepare_dataloader.py index 193ec384..8a7e3d1e 100644 --- a/fastNLP/core/dataloaders/prepare_dataloader.py +++ b/fastNLP/core/dataloaders/prepare_dataloader.py @@ -10,7 +10,7 @@ from ..samplers import RandomBatchSampler, RandomSampler from .torch_dataloader import prepare_torch_dataloader from .paddle_dataloader import prepare_paddle_dataloader from .jittor_dataloader import prepare_jittor_dataloader -from ...envs import FASTNLP_BACKEND, SUPPORT_BACKENDS, _module_available +from ...envs import FASTNLP_BACKEND, SUPPORT_BACKENDS from ..log import logger diff --git a/fastNLP/core/dataset/dataset.py b/fastNLP/core/dataset/dataset.py index 83e83ac9..6e40d7ef 100644 --- a/fastNLP/core/dataset/dataset.py +++ b/fastNLP/core/dataset/dataset.py @@ -451,8 +451,8 @@ class DataSet: apply_out = self._apply_process(num_proc, func, progress_desc=progress_desc, show_progress_bar=show_progress_bar, _apply_field=field_name) # 只检测第一个数据是否为dict类型，若是则默认所有返回值为dict；否则报错。 - if not isinstance(apply_out[0], dict): - raise Exception("The result of func is not a dict") + if not isinstance(apply_out[0], Mapping): + raise Exception(f"The result of func is not a Mapping, but a {type(apply_out[0])}") for key, value in apply_out[0].items(): results[key] = [value] diff --git a/fastNLP/core/drivers/torch_driver/initialize_torch_driver.py b/fastNLP/core/drivers/torch_driver/initialize_torch_driver.py index 1ca83c09..025744bb 100644 --- a/fastNLP/core/drivers/torch_driver/initialize_torch_driver.py +++ b/fastNLP/core/drivers/torch_driver/initialize_torch_driver.py @@ -55,8 +55,8 @@ def initialize_torch_driver(driver: str, device: Optional[Union[str, "torch.devi elif each < 0: raise ValueError("When parameter `device` is 'Sequence' type, the value in it should be bigger than 0.") elif each >= _could_use_device_num: - raise ValueError("When parameter `device` is 'Sequence' type, the value in it should not be bigger than" - " the available gpu number.") + raise ValueError(f"When parameter `device` is 'Sequence' type, the value in it should not be bigger than" + f" the available gpu number:{_could_use_device_num}.") device = [torch.device(f"cuda:{w}") for w in device] elif device is not None and not isinstance(device, torch.device): raise ValueError("Parameter `device` is wrong type, please check our documentation for the right use.") diff --git a/fastNLP/core/drivers/torch_driver/torch_driver.py b/fastNLP/core/drivers/torch_driver/torch_driver.py index c74c79ed..5aee15e9 100644 --- a/fastNLP/core/drivers/torch_driver/torch_driver.py +++ b/fastNLP/core/drivers/torch_driver/torch_driver.py @@ -167,6 +167,12 @@ class TorchDriver(Driver): """ model = self.unwrap_model() res = torch.load(filepath, map_location='cpu') + if isinstance(res, dict) and only_state_dict is False: + logger.rank_zero_warning(f"It seems like that {filepath} only contains state, you may need to use " + f"`only_state_dict=True`") + elif not isinstance(res, dict) and only_state_dict is True: + logger.rank_zero_warning(f"It seems like that {filepath} is not state, you may need to use " + f"`only_state_dict=False`") if only_state_dict: model.load_state_dict(res) else: diff --git a/tests/core/collators/test_collator.py b/tests/core/collators/test_collator.py index ae219793..09ec4af8 100644 --- a/tests/core/collators/test_collator.py +++ b/tests/core/collators/test_collator.py @@ -334,9 +334,9 @@ def test_torch_dl(): dl = TorchDataLoader(ds, batch_size=2) batch = next(iter(dl)) assert 'x' in batch and 'y' in batch and 'z' in batch and 'i' in batch and 'j' in batch - assert isinstance(batch['z'], torch.Tensor) + assert isinstance(batch['z'], torch.FloatTensor) assert isinstance(batch['j'], list) - assert isinstance(batch['i']['j'], torch.Tensor) + assert isinstance(batch['i']['j'], torch.LongTensor) dl.set_ignore('x') batch = next(iter(dl)) From 49cf176e742ae3993e08b944e07a15e76016a24a Mon Sep 17 00:00:00 2001 From: x54-729 <17307130121@fudan.edu.cn> Date: Mon, 9 May 2022 14:54:30 +0000 Subject: [PATCH 05/15] =?UTF-8?q?=E5=AE=8C=E5=96=84=E6=96=87=E6=A1=A3?= =?UTF-8?q?=EF=BC=9B=E6=9B=B4=E6=94=B9=E6=96=87=E6=A1=A3=E8=AE=BE=E7=BD=AE?= =?UTF-8?q?=EF=BC=8C=E7=8E=B0=E5=9C=A8=E5=8F=AF=E4=BB=A5=E5=B1=95=E7=A4=BA?= =?UTF-8?q?=E5=8F=82=E6=95=B0=E7=9A=84=E9=BB=98=E8=AE=A4=E5=80=BC=E4=BA=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/Makefile | 5 +- docs/source/conf.py | 11 +- docs/source/fastNLP.core.callbacks.rst | 4 +- ...fastNLP.core.callbacks.torch_callbacks.rst | 2 +- .../source/fastNLP.core.collators.padders.rst | 2 +- docs/source/fastNLP.core.collators.rst | 4 +- .../source/fastNLP.core.controllers.loops.rst | 2 +- docs/source/fastNLP.core.controllers.rst | 4 +- .../source/fastNLP.core.controllers.utils.rst | 2 +- ...NLP.core.dataloaders.jittor_dataloader.rst | 2 +- ...NLP.core.dataloaders.paddle_dataloader.rst | 2 +- ...LP.core.dataloaders.prepare_dataloader.rst | 7 + docs/source/fastNLP.core.dataloaders.rst | 5 +- ...tNLP.core.dataloaders.torch_dataloader.rst | 2 +- docs/source/fastNLP.core.dataset.rst | 2 +- .../fastNLP.core.drivers.jittor_driver.rst | 2 +- .../fastNLP.core.drivers.paddle_driver.rst | 2 +- docs/source/fastNLP.core.drivers.rst | 4 +- .../fastNLP.core.drivers.torch_driver.rst | 2 +- docs/source/fastNLP.core.log.rst | 2 +- ...LP.core.metrics.backend.jittor_backend.rst | 2 +- ...LP.core.metrics.backend.paddle_backend.rst | 2 +- docs/source/fastNLP.core.metrics.backend.rst | 4 +- ...NLP.core.metrics.backend.torch_backend.rst | 2 +- docs/source/fastNLP.core.metrics.rst | 4 +- docs/source/fastNLP.core.rst | 4 +- docs/source/fastNLP.core.samplers.rst | 2 +- docs/source/fastNLP.core.utils.rst | 2 +- docs/source/fastNLP.envs.rst | 2 +- docs/source/fastNLP.io.loader.rst | 2 +- docs/source/fastNLP.io.pipe.rst | 2 +- docs/source/fastNLP.io.rst | 4 +- docs/source/fastNLP.rst | 2 +- docs/source/modules.rst | 2 +- fastNLP/core/callbacks/utils.py | 4 +- .../core/dataloaders/paddle_dataloader/fdl.py | 4 +- .../core/dataloaders/torch_dataloader/fdl.py | 4 +- fastNLP/core/dataset/dataset.py | 19 ++- fastNLP/core/utils/dummy_class.py | 2 +- fastNLP/core/utils/rich_progress.py | 5 +- fastNLP/core/utils/utils.py | 159 +++++++++--------- fastNLP/envs/utils.py | 8 +- 42 files changed, 167 insertions(+), 142 deletions(-) create mode 100644 docs/source/fastNLP.core.dataloaders.prepare_dataloader.rst diff --git a/docs/Makefile b/docs/Makefile index 9f807ae2..d6c4f6b6 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -16,7 +16,7 @@ help: @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) apidoc: - $(SPHINXAPIDOC) -efM -d 6 -o source ../$(SPHINXPROJ) $(SPHINXEXCLUDE) + $(SPHINXAPIDOC) -efM -o source ../$(SPHINXPROJ) $(SPHINXEXCLUDE) server: cd build/html && python -m http.server $(PORT) @@ -24,6 +24,9 @@ server: delete: rm -f source/$(SPHINXPROJ).* source/modules.rst && rm -rf build +web: + make html && make server + dev: make delete && make apidoc && make html && make server diff --git a/docs/source/conf.py b/docs/source/conf.py index 115448ed..2ed8ac96 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -42,7 +42,8 @@ extensions = [ 'sphinx.ext.viewcode', 'sphinx.ext.autosummary', 'sphinx.ext.mathjax', - 'sphinx.ext.todo' + 'sphinx.ext.todo', + 'sphinx_autodoc_typehints' ] autodoc_default_options = { @@ -53,8 +54,10 @@ autodoc_default_options = { add_module_names = False autosummary_ignore_module_all = False -autodoc_typehints = "description" +# autodoc_typehints = "description" autoclass_content = "class" +typehints_fully_qualified = False +typehints_defaults = "comma" # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -168,8 +171,8 @@ texinfo_documents = [ # -- Extension configuration ------------------------------------------------- def maybe_skip_member(app, what, name, obj, skip, options): - # if obj.__doc__ is None: - # return True + if obj.__doc__ is None: + return True if name == "__init__": return False if name.startswith("_"): diff --git a/docs/source/fastNLP.core.callbacks.rst b/docs/source/fastNLP.core.callbacks.rst index a3450110..0f3f93ac 100644 --- a/docs/source/fastNLP.core.callbacks.rst +++ b/docs/source/fastNLP.core.callbacks.rst @@ -10,7 +10,7 @@ Subpackages ----------- .. toctree:: - :maxdepth: 6 + :maxdepth: 4 fastNLP.core.callbacks.torch_callbacks @@ -18,7 +18,7 @@ Submodules ---------- .. toctree:: - :maxdepth: 6 + :maxdepth: 4 fastNLP.core.callbacks.callback fastNLP.core.callbacks.callback_event diff --git a/docs/source/fastNLP.core.callbacks.torch_callbacks.rst b/docs/source/fastNLP.core.callbacks.torch_callbacks.rst index 193f46d3..6f00f6f7 100644 --- a/docs/source/fastNLP.core.callbacks.torch_callbacks.rst +++ b/docs/source/fastNLP.core.callbacks.torch_callbacks.rst @@ -10,7 +10,7 @@ Submodules ---------- .. toctree:: - :maxdepth: 6 + :maxdepth: 4 fastNLP.core.callbacks.torch_callbacks.torch_grad_clip_callback fastNLP.core.callbacks.torch_callbacks.torch_lr_sched_callback diff --git a/docs/source/fastNLP.core.collators.padders.rst b/docs/source/fastNLP.core.collators.padders.rst index 0ee61a26..6f40becb 100644 --- a/docs/source/fastNLP.core.collators.padders.rst +++ b/docs/source/fastNLP.core.collators.padders.rst @@ -10,7 +10,7 @@ Submodules ---------- .. toctree:: - :maxdepth: 6 + :maxdepth: 4 fastNLP.core.collators.padders.exceptions fastNLP.core.collators.padders.get_padder diff --git a/docs/source/fastNLP.core.collators.rst b/docs/source/fastNLP.core.collators.rst index 1210e8b3..22259c12 100644 --- a/docs/source/fastNLP.core.collators.rst +++ b/docs/source/fastNLP.core.collators.rst @@ -10,7 +10,7 @@ Subpackages ----------- .. toctree:: - :maxdepth: 6 + :maxdepth: 4 fastNLP.core.collators.padders @@ -18,7 +18,7 @@ Submodules ---------- .. toctree:: - :maxdepth: 6 + :maxdepth: 4 fastNLP.core.collators.collator fastNLP.core.collators.packer_unpacker diff --git a/docs/source/fastNLP.core.controllers.loops.rst b/docs/source/fastNLP.core.controllers.loops.rst index 8db384f7..39879148 100644 --- a/docs/source/fastNLP.core.controllers.loops.rst +++ b/docs/source/fastNLP.core.controllers.loops.rst @@ -10,7 +10,7 @@ Submodules ---------- .. toctree:: - :maxdepth: 6 + :maxdepth: 4 fastNLP.core.controllers.loops.evaluate_batch_loop fastNLP.core.controllers.loops.loop diff --git a/docs/source/fastNLP.core.controllers.rst b/docs/source/fastNLP.core.controllers.rst index daef5f3b..9440fbe4 100644 --- a/docs/source/fastNLP.core.controllers.rst +++ b/docs/source/fastNLP.core.controllers.rst @@ -10,7 +10,7 @@ Subpackages ----------- .. toctree:: - :maxdepth: 6 + :maxdepth: 4 fastNLP.core.controllers.loops fastNLP.core.controllers.utils @@ -19,7 +19,7 @@ Submodules ---------- .. toctree:: - :maxdepth: 6 + :maxdepth: 4 fastNLP.core.controllers.evaluator fastNLP.core.controllers.trainer diff --git a/docs/source/fastNLP.core.controllers.utils.rst b/docs/source/fastNLP.core.controllers.utils.rst index ca8a7307..f7bcc38c 100644 --- a/docs/source/fastNLP.core.controllers.utils.rst +++ b/docs/source/fastNLP.core.controllers.utils.rst @@ -10,7 +10,7 @@ Submodules ---------- .. toctree:: - :maxdepth: 6 + :maxdepth: 4 fastNLP.core.controllers.utils.state fastNLP.core.controllers.utils.utils diff --git a/docs/source/fastNLP.core.dataloaders.jittor_dataloader.rst b/docs/source/fastNLP.core.dataloaders.jittor_dataloader.rst index d7a7a8dc..78d90c46 100644 --- a/docs/source/fastNLP.core.dataloaders.jittor_dataloader.rst +++ b/docs/source/fastNLP.core.dataloaders.jittor_dataloader.rst @@ -10,6 +10,6 @@ Submodules ---------- .. toctree:: - :maxdepth: 6 + :maxdepth: 4 fastNLP.core.dataloaders.jittor_dataloader.fdl diff --git a/docs/source/fastNLP.core.dataloaders.paddle_dataloader.rst b/docs/source/fastNLP.core.dataloaders.paddle_dataloader.rst index 428a339e..dc4481d2 100644 --- a/docs/source/fastNLP.core.dataloaders.paddle_dataloader.rst +++ b/docs/source/fastNLP.core.dataloaders.paddle_dataloader.rst @@ -10,6 +10,6 @@ Submodules ---------- .. toctree:: - :maxdepth: 6 + :maxdepth: 4 fastNLP.core.dataloaders.paddle_dataloader.fdl diff --git a/docs/source/fastNLP.core.dataloaders.prepare_dataloader.rst b/docs/source/fastNLP.core.dataloaders.prepare_dataloader.rst new file mode 100644 index 00000000..ac8c8c20 --- /dev/null +++ b/docs/source/fastNLP.core.dataloaders.prepare_dataloader.rst @@ -0,0 +1,7 @@ +fastNLP.core.dataloaders.prepare\_dataloader module +=================================================== + +.. automodule:: fastNLP.core.dataloaders.prepare_dataloader + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/fastNLP.core.dataloaders.rst b/docs/source/fastNLP.core.dataloaders.rst index a9bd51fa..e8c6b799 100644 --- a/docs/source/fastNLP.core.dataloaders.rst +++ b/docs/source/fastNLP.core.dataloaders.rst @@ -10,7 +10,7 @@ Subpackages ----------- .. toctree:: - :maxdepth: 6 + :maxdepth: 4 fastNLP.core.dataloaders.jittor_dataloader fastNLP.core.dataloaders.paddle_dataloader @@ -20,7 +20,8 @@ Submodules ---------- .. toctree:: - :maxdepth: 6 + :maxdepth: 4 fastNLP.core.dataloaders.mix_dataloader + fastNLP.core.dataloaders.prepare_dataloader fastNLP.core.dataloaders.utils diff --git a/docs/source/fastNLP.core.dataloaders.torch_dataloader.rst b/docs/source/fastNLP.core.dataloaders.torch_dataloader.rst index f631571e..c9acca23 100644 --- a/docs/source/fastNLP.core.dataloaders.torch_dataloader.rst +++ b/docs/source/fastNLP.core.dataloaders.torch_dataloader.rst @@ -10,6 +10,6 @@ Submodules ---------- .. toctree:: - :maxdepth: 6 + :maxdepth: 4 fastNLP.core.dataloaders.torch_dataloader.fdl diff --git a/docs/source/fastNLP.core.dataset.rst b/docs/source/fastNLP.core.dataset.rst index e3ceff77..dc36250a 100644 --- a/docs/source/fastNLP.core.dataset.rst +++ b/docs/source/fastNLP.core.dataset.rst @@ -10,7 +10,7 @@ Submodules ---------- .. toctree:: - :maxdepth: 6 + :maxdepth: 4 fastNLP.core.dataset.dataset fastNLP.core.dataset.field diff --git a/docs/source/fastNLP.core.drivers.jittor_driver.rst b/docs/source/fastNLP.core.drivers.jittor_driver.rst index df32665b..7ec101c7 100644 --- a/docs/source/fastNLP.core.drivers.jittor_driver.rst +++ b/docs/source/fastNLP.core.drivers.jittor_driver.rst @@ -10,7 +10,7 @@ Submodules ---------- .. toctree:: - :maxdepth: 6 + :maxdepth: 4 fastNLP.core.drivers.jittor_driver.initialize_jittor_driver fastNLP.core.drivers.jittor_driver.jittor_driver diff --git a/docs/source/fastNLP.core.drivers.paddle_driver.rst b/docs/source/fastNLP.core.drivers.paddle_driver.rst index 91038646..0f115eb5 100644 --- a/docs/source/fastNLP.core.drivers.paddle_driver.rst +++ b/docs/source/fastNLP.core.drivers.paddle_driver.rst @@ -10,7 +10,7 @@ Submodules ---------- .. toctree:: - :maxdepth: 6 + :maxdepth: 4 fastNLP.core.drivers.paddle_driver.dist_utils fastNLP.core.drivers.paddle_driver.fleet diff --git a/docs/source/fastNLP.core.drivers.rst b/docs/source/fastNLP.core.drivers.rst index 3ac36f71..bb168c76 100644 --- a/docs/source/fastNLP.core.drivers.rst +++ b/docs/source/fastNLP.core.drivers.rst @@ -10,7 +10,7 @@ Subpackages ----------- .. toctree:: - :maxdepth: 6 + :maxdepth: 4 fastNLP.core.drivers.jittor_driver fastNLP.core.drivers.paddle_driver @@ -20,7 +20,7 @@ Submodules ---------- .. toctree:: - :maxdepth: 6 + :maxdepth: 4 fastNLP.core.drivers.choose_driver fastNLP.core.drivers.driver diff --git a/docs/source/fastNLP.core.drivers.torch_driver.rst b/docs/source/fastNLP.core.drivers.torch_driver.rst index 65f4ca1a..9a0109a2 100644 --- a/docs/source/fastNLP.core.drivers.torch_driver.rst +++ b/docs/source/fastNLP.core.drivers.torch_driver.rst @@ -10,7 +10,7 @@ Submodules ---------- .. toctree:: - :maxdepth: 6 + :maxdepth: 4 fastNLP.core.drivers.torch_driver.ddp fastNLP.core.drivers.torch_driver.dist_utils diff --git a/docs/source/fastNLP.core.log.rst b/docs/source/fastNLP.core.log.rst index e52f9eb7..6cd67753 100644 --- a/docs/source/fastNLP.core.log.rst +++ b/docs/source/fastNLP.core.log.rst @@ -10,7 +10,7 @@ Submodules ---------- .. toctree:: - :maxdepth: 6 + :maxdepth: 4 fastNLP.core.log.handler fastNLP.core.log.highlighter diff --git a/docs/source/fastNLP.core.metrics.backend.jittor_backend.rst b/docs/source/fastNLP.core.metrics.backend.jittor_backend.rst index 9b76aee3..6ce8b0d4 100644 --- a/docs/source/fastNLP.core.metrics.backend.jittor_backend.rst +++ b/docs/source/fastNLP.core.metrics.backend.jittor_backend.rst @@ -10,6 +10,6 @@ Submodules ---------- .. toctree:: - :maxdepth: 6 + :maxdepth: 4 fastNLP.core.metrics.backend.jittor_backend.backend diff --git a/docs/source/fastNLP.core.metrics.backend.paddle_backend.rst b/docs/source/fastNLP.core.metrics.backend.paddle_backend.rst index fb4ec69d..d932d4e5 100644 --- a/docs/source/fastNLP.core.metrics.backend.paddle_backend.rst +++ b/docs/source/fastNLP.core.metrics.backend.paddle_backend.rst @@ -10,6 +10,6 @@ Submodules ---------- .. toctree:: - :maxdepth: 6 + :maxdepth: 4 fastNLP.core.metrics.backend.paddle_backend.backend diff --git a/docs/source/fastNLP.core.metrics.backend.rst b/docs/source/fastNLP.core.metrics.backend.rst index 52ca7958..5a8cf4ad 100644 --- a/docs/source/fastNLP.core.metrics.backend.rst +++ b/docs/source/fastNLP.core.metrics.backend.rst @@ -10,7 +10,7 @@ Subpackages ----------- .. toctree:: - :maxdepth: 6 + :maxdepth: 4 fastNLP.core.metrics.backend.jittor_backend fastNLP.core.metrics.backend.paddle_backend @@ -20,7 +20,7 @@ Submodules ---------- .. toctree:: - :maxdepth: 6 + :maxdepth: 4 fastNLP.core.metrics.backend.auto_backend fastNLP.core.metrics.backend.backend diff --git a/docs/source/fastNLP.core.metrics.backend.torch_backend.rst b/docs/source/fastNLP.core.metrics.backend.torch_backend.rst index 07beae73..f01efe88 100644 --- a/docs/source/fastNLP.core.metrics.backend.torch_backend.rst +++ b/docs/source/fastNLP.core.metrics.backend.torch_backend.rst @@ -10,6 +10,6 @@ Submodules ---------- .. toctree:: - :maxdepth: 6 + :maxdepth: 4 fastNLP.core.metrics.backend.torch_backend.backend diff --git a/docs/source/fastNLP.core.metrics.rst b/docs/source/fastNLP.core.metrics.rst index e2770769..8ad6f729 100644 --- a/docs/source/fastNLP.core.metrics.rst +++ b/docs/source/fastNLP.core.metrics.rst @@ -10,7 +10,7 @@ Subpackages ----------- .. toctree:: - :maxdepth: 6 + :maxdepth: 4 fastNLP.core.metrics.backend @@ -18,7 +18,7 @@ Submodules ---------- .. toctree:: - :maxdepth: 6 + :maxdepth: 4 fastNLP.core.metrics.accuracy fastNLP.core.metrics.classify_f1_pre_rec_metric diff --git a/docs/source/fastNLP.core.rst b/docs/source/fastNLP.core.rst index d71ffaf3..57dac16a 100644 --- a/docs/source/fastNLP.core.rst +++ b/docs/source/fastNLP.core.rst @@ -10,7 +10,7 @@ Subpackages ----------- .. toctree:: - :maxdepth: 6 + :maxdepth: 4 fastNLP.core.callbacks fastNLP.core.collators @@ -27,6 +27,6 @@ Submodules ---------- .. toctree:: - :maxdepth: 6 + :maxdepth: 4 fastNLP.core.vocabulary diff --git a/docs/source/fastNLP.core.samplers.rst b/docs/source/fastNLP.core.samplers.rst index f1b7be4c..9ccd9b59 100644 --- a/docs/source/fastNLP.core.samplers.rst +++ b/docs/source/fastNLP.core.samplers.rst @@ -10,7 +10,7 @@ Submodules ---------- .. toctree:: - :maxdepth: 6 + :maxdepth: 4 fastNLP.core.samplers.conversion_utils fastNLP.core.samplers.mix_sampler diff --git a/docs/source/fastNLP.core.utils.rst b/docs/source/fastNLP.core.utils.rst index a63ed1db..2d682010 100644 --- a/docs/source/fastNLP.core.utils.rst +++ b/docs/source/fastNLP.core.utils.rst @@ -10,7 +10,7 @@ Submodules ---------- .. toctree:: - :maxdepth: 6 + :maxdepth: 4 fastNLP.core.utils.cache_results fastNLP.core.utils.dummy_class diff --git a/docs/source/fastNLP.envs.rst b/docs/source/fastNLP.envs.rst index 4c95ccfe..2e642ff7 100644 --- a/docs/source/fastNLP.envs.rst +++ b/docs/source/fastNLP.envs.rst @@ -10,7 +10,7 @@ Submodules ---------- .. toctree:: - :maxdepth: 6 + :maxdepth: 4 fastNLP.envs.distributed fastNLP.envs.env diff --git a/docs/source/fastNLP.io.loader.rst b/docs/source/fastNLP.io.loader.rst index 13bd5fe9..bd91b795 100644 --- a/docs/source/fastNLP.io.loader.rst +++ b/docs/source/fastNLP.io.loader.rst @@ -10,7 +10,7 @@ Submodules ---------- .. toctree:: - :maxdepth: 6 + :maxdepth: 4 fastNLP.io.loader.classification fastNLP.io.loader.conll diff --git a/docs/source/fastNLP.io.pipe.rst b/docs/source/fastNLP.io.pipe.rst index d8cf306e..9ad7e539 100644 --- a/docs/source/fastNLP.io.pipe.rst +++ b/docs/source/fastNLP.io.pipe.rst @@ -10,7 +10,7 @@ Submodules ---------- .. toctree:: - :maxdepth: 6 + :maxdepth: 4 fastNLP.io.pipe.classification fastNLP.io.pipe.conll diff --git a/docs/source/fastNLP.io.rst b/docs/source/fastNLP.io.rst index 4fab1696..5f025bba 100644 --- a/docs/source/fastNLP.io.rst +++ b/docs/source/fastNLP.io.rst @@ -10,7 +10,7 @@ Subpackages ----------- .. toctree:: - :maxdepth: 6 + :maxdepth: 4 fastNLP.io.loader fastNLP.io.pipe @@ -19,7 +19,7 @@ Submodules ---------- .. toctree:: - :maxdepth: 6 + :maxdepth: 4 fastNLP.io.data_bundle fastNLP.io.embed_loader diff --git a/docs/source/fastNLP.rst b/docs/source/fastNLP.rst index bee33e72..726eb9c6 100644 --- a/docs/source/fastNLP.rst +++ b/docs/source/fastNLP.rst @@ -10,7 +10,7 @@ Subpackages ----------- .. toctree:: - :maxdepth: 6 + :maxdepth: 4 fastNLP.core fastNLP.envs diff --git a/docs/source/modules.rst b/docs/source/modules.rst index 5515520a..e9a92cb7 100644 --- a/docs/source/modules.rst +++ b/docs/source/modules.rst @@ -2,6 +2,6 @@ fastNLP ======= .. toctree:: - :maxdepth: 6 + :maxdepth: 4 fastNLP diff --git a/fastNLP/core/callbacks/utils.py b/fastNLP/core/callbacks/utils.py index c3f8275a..865a1fc7 100644 --- a/fastNLP/core/callbacks/utils.py +++ b/fastNLP/core/callbacks/utils.py @@ -8,8 +8,8 @@ from fastNLP.core.utils.utils import _get_fun_msg def _get_monitor_value(monitor: Union[callable, str], real_monitor: Optional[str], res: dict) ->Tuple[str, float]: """ - 从res中寻找 monitor 并返回。如果 monitor 没找到则尝试用 _real_monitor ,若 _real_monitor 为 None 则尝试使用 monitor 的值进行 - 匹配。 + 从 ``res`` 中寻找 ``monitor`` 并返回。如果 ``monitor`` 没找到则尝试用 ``_real_monitor`` ,若 ``_real_monitor`` 为 ``None`` + 则尝试使用 ``monitor`` 的值进行匹配。 :param monitor: :param real_monitor: diff --git a/fastNLP/core/dataloaders/paddle_dataloader/fdl.py b/fastNLP/core/dataloaders/paddle_dataloader/fdl.py index 5c5e3bef..342a6c19 100644 --- a/fastNLP/core/dataloaders/paddle_dataloader/fdl.py +++ b/fastNLP/core/dataloaders/paddle_dataloader/fdl.py @@ -162,9 +162,9 @@ class PaddleDataLoader(DataLoader): def get_batch_indices(self) -> List[int]: """ - 获取当前 batch 的 idx + 获取当前 ``batch`` 中每条数据对应的索引。 - :return: + :return: 当前 ``batch`` 数据的索引 """ return self.cur_batch_indices diff --git a/fastNLP/core/dataloaders/torch_dataloader/fdl.py b/fastNLP/core/dataloaders/torch_dataloader/fdl.py index 6a9e4af9..1f737467 100644 --- a/fastNLP/core/dataloaders/torch_dataloader/fdl.py +++ b/fastNLP/core/dataloaders/torch_dataloader/fdl.py @@ -170,9 +170,9 @@ class TorchDataLoader(DataLoader): def get_batch_indices(self) -> List[int]: """ - 获取当前 batch 的 idx + 获取当前 ``batch`` 中每条数据对应的索引。 - :return: + :return: 当前 ``batch`` 数据的索引 """ return self.cur_batch_indices diff --git a/fastNLP/core/dataset/dataset.py b/fastNLP/core/dataset/dataset.py index 83e83ac9..7cba5a8c 100644 --- a/fastNLP/core/dataset/dataset.py +++ b/fastNLP/core/dataset/dataset.py @@ -400,15 +400,16 @@ class DataSet: new_field_name: str = None, num_proc: int = 0, progress_desc: str = None, show_progress_bar: bool = True): r""" - 将 DataSet 中的每个 instance 中的名为 `field_name` 的 field 传给 func，并获取它的返回值。 - - :param field_name: 传入 func 的是哪个 field。 - :param func: input是 instance 中名为 `field_name` 的 field 的内容。 - :param new_field_name: 将 func 返回的内容放入到 `new_field_name` 这个 field 中，如果名称与已有的 field 相同，则覆 - 盖之前的 field。如果为 None 则不创建新的 field。 - :param num_proc: 进程的数量。请注意，由于python语言的特性，多少进程就会导致多少倍内存的增长。 - :param progress_desc: progress_desc 的值，默认为 Main - :param show_progress_bar: 是否展示进度条，默认展示进度条 + 将 :class:`~DataSet` 每个 ``instance`` 中为 ``field_name`` 的 ``field`` 传给函数 ``func``，并获取函数的返回值。 + + :param field_name: 传入 ``func`` 的 ``field`` 名称。 + :param func: 一个函数，其输入是 ``instance`` 中名为 ``field_name`` 的 ``field`` 的内容。 + :param new_field_name: 将 ``func`` 返回的内容放入到 ``new_field_name`` 对应的 ``field`` 中，如果名称与已有的 ``field`` 相同 + 则进行覆盖。如果为 ``None`` 则不会覆盖和创建 ``field`` 。 + :param num_proc: 使用进程的数量。请注意，由于 ``python`` 语言的特性，使用了多少进程就会导致多少倍内存的增长。 + :param progress_desc: 进度条的描述字符，默认为 ``Main``。 + :param show_progress_bar: 是否展示进度条；默认为展示。 + :return: 从函数 ``func`` 中得到的返回值。 """ assert len(self) != 0, "Null DataSet cannot use apply_field()." if not self.has_field(field_name=field_name): diff --git a/fastNLP/core/utils/dummy_class.py b/fastNLP/core/utils/dummy_class.py index 42200cbb..e7596607 100644 --- a/fastNLP/core/utils/dummy_class.py +++ b/fastNLP/core/utils/dummy_class.py @@ -1,4 +1,4 @@ -import functools +__all__ = [] class DummyClass: def __init__(self, *args, **kwargs): diff --git a/fastNLP/core/utils/rich_progress.py b/fastNLP/core/utils/rich_progress.py index 4799765f..02a30c26 100644 --- a/fastNLP/core/utils/rich_progress.py +++ b/fastNLP/core/utils/rich_progress.py @@ -1,7 +1,6 @@ """ -该文件用于为fastNLP提供一个统一的progress bar管理，通过共用一个Task对象，trainer中的progress bar和evaluation中的progress bar才能 - 不冲突 - +该文件用于为 ``fastNLP`` 提供一个统一的 ``progress bar`` 管理，通过共用一个``Task`` 对象， :class:`~fastNLP.core.Trainer` 中 +的 ``progress bar`` 和 :class:`~fastNLP.core.Evaluator` 中的 ``progress bar`` 才能不冲突 """ import sys from typing import Any, Union, Optional diff --git a/fastNLP/core/utils/utils.py b/fastNLP/core/utils/utils.py index b07d8b82..ec7a8b47 100644 --- a/fastNLP/core/utils/utils.py +++ b/fastNLP/core/utils/utils.py @@ -10,10 +10,6 @@ from typing import Callable, List, Any, Dict, AnyStr, Union, Mapping, Sequence from typing import Tuple, Optional from time import sleep -try: - from typing import Literal, Final -except ImportError: - from typing_extensions import Literal, Final import os from contextlib import contextmanager from functools import wraps @@ -22,7 +18,6 @@ import numpy as np from pathlib import Path from fastNLP.core.log import logger -from ...envs import SUPPORT_BACKENDS __all__ = [ @@ -43,10 +38,10 @@ __all__ = [ def get_fn_arg_names(fn: Callable) -> List[str]: r""" - 返回一个函数的所有参数的名字； + 返回一个函数所有参数的名字 - :param fn: 需要查询的函数； - :return: 一个列表，其中的元素则是查询函数的参数的字符串名字； + :param fn: 需要查询的函数 + :return: 一个列表，其中的元素是函数 ``fn`` 参数的字符串名字 """ return list(inspect.signature(fn).parameters) @@ -54,24 +49,18 @@ def get_fn_arg_names(fn: Callable) -> List[str]: def auto_param_call(fn: Callable, *args, signature_fn: Optional[Callable] = None, mapping: Optional[Dict[AnyStr, AnyStr]] = None) -> Any: r""" - 该函数会根据输入函数的形参名从*args（因此都需要是dict类型）中找到匹配的值进行调用，如果传入的数据与fn的形参不匹配，可以通过mapping - 参数进行转换。mapping参数中的一对（key，value）表示以这个key在*args中找到值，并将这个值传递给形参名为value的参数。 + 该函数会根据输入函数的形参名从 ``*args`` （因此都需要是 ``dict`` 类型）中找到匹配的值进行调用，如果传入的数据与 ``fn`` 的形参不匹配，可以通过 + ``mapping`` 参数进行转换。``mapping`` 参数中的一对 ``(key, value)`` 表示在 ``*args`` 中找到 ``key`` 对应的值，并将这个值传递给形参中名为 + ``value`` 的参数。 - 1.该函数用来提供给用户根据字符串匹配从而实现自动调用； - 2.注意 mapping 默认为 None，如果你希望指定输入和运行函数的参数的对应方式，那么你应当让 mapping 为一个这样的字典传入进来； - 如果 mapping 不为 None，那么我们一定会先使用 mapping 将输入的字典的 keys 修改过来，因此请务必亲自检查 mapping 的正确性； - 3.如果输入的函数的参数有默认值，那么如果之后的输入中没有该参数对应的值，我们就会使用该参数对应的默认值，否则也会使用之后的输入的值； - 4.如果输入的函数是一个 `partial` 函数，情况同 '3.'，即和默认参数的情况相同； - - :param fn: 用来进行实际计算的函数，其参数可以包含有默认值； - :param args: 一系列的位置参数，应当为一系列的字典，我们需要从这些输入中提取 `fn` 计算所需要的实际参数； - :param signature_fn: 函数，用来替换 `fn` 的函数签名，如果该参数不为 None，那么我们首先会从该函数中提取函数签名，然后通过该函数签名提取 - 参数值后，再传给 `fn` 进行实际的运算； - :param mapping: 一个字典，用来更改其前面的字典的键值； - - :return: 返回 `fn` 运行的结果； + 1. 该函数用来提供给用户根据字符串匹配从而实现自动调用； + 2. 注意 ``mapping`` 默认为 ``None``，如果你希望指定输入和运行函数的参数的对应方式，那么你应当让 ``mapping`` 为一个字典传入进来； + 如果 ``mapping`` 不为 ``None``，那么我们一定会先使用 ``mapping`` 将输入的字典的 ``keys`` 修改过来，因此请务必亲自检查 ``mapping`` 的正确性； + 3. 如果输入的函数的参数有默认值，那么如果之后的输入中没有该参数对应的值，我们就会使用该参数对应的默认值，否则也会使用之后的输入的值； + 4. 如果输入的函数是一个 ``partial`` 函数，情况同第三点，即和默认参数的情况相同； Examples:: + >>> # 1 >>> loss_fn = CrossEntropyLoss() # 如果其需要的参数为 def CrossEntropyLoss(y, pred)； >>> batch = {"x": 20, "y": 1} @@ -84,6 +73,14 @@ def auto_param_call(fn: Callable, *args, signature_fn: Optional[Callable] = None >>> print(auto_param_call(test_fn, {"x": 10}, {"y": 20, "a": 30})) # res: 70 >>> print(auto_param_call(partial(test_fn, a=100), {"x": 10}, {"y": 20})) # res: 140 >>> print(auto_param_call(partial(test_fn, a=100), {"x": 10}, {"y": 20, "a": 200})) # res: 240 + + :param fn: 用来进行实际计算的函数，其参数可以包含有默认值； + :param args: 一系列的位置参数，应当为一系列的字典，我们需要从这些输入中提取 ``fn`` 计算所需要的实际参数； + :param signature_fn: 函数，用来替换 ``fn`` 的函数签名，如果该参数不为 ``None``，那么我们首先会从该函数中提取函数签名，然后通过该函数签名提取 + 参数值后，再传给 ``fn`` 进行实际的运算； + :param mapping: 一个字典，用来更改其前面的字典的键值； + + :return: 返回 ``fn`` 运行的结果； """ if signature_fn is not None: @@ -226,13 +223,13 @@ def _check_valid_parameters_number(fn, expected_params:List[str], fn_name=None): def check_user_specific_params(user_params: Dict, fn: Callable): """ - 该函数使用用户的输入来对指定函数的参数进行赋值； - 主要用于一些用户无法直接调用函数的情况； - 该函数主要的作用在于帮助检查用户对使用函数 fn 的参数输入是否有误； + 该函数使用用户的输入来对指定函数的参数进行赋值，主要用于一些用户无法直接调用函数的情况； + 该函数主要的作用在于帮助检查用户对使用函数 ``fn`` 的参数输入是否有误； - :param user_params: 用户指定的参数的值，应当是一个字典，其中 key 表示每一个参数的名字，value 为每一个参数应当的值； - :param fn: 会被调用的函数； - :return: 返回一个字典，其中为在之后调用函数 fn 时真正会被传进去的参数的值； + :param user_params: 用户指定的参数的值，应当是一个字典，其中 ``key`` 表示每一个参数的名字， + ``value`` 为每一个参数的值； + :param fn: 将要被调用的函数； + :return: 返回一个字典，其中为在之后调用函数 ``fn`` 时真正会被传进去的参数的值； """ fn_arg_names = get_fn_arg_names(fn) @@ -243,6 +240,9 @@ def check_user_specific_params(user_params: Dict, fn: Callable): def dataclass_to_dict(data: "dataclasses.dataclass") -> Dict: + """ + 将传入的 `dataclass` 实例转换为字典。 + """ if not is_dataclass(data): raise TypeError(f"Parameter `data` can only be `dataclass` type instead of {type(data)}.") _dict = dict() @@ -253,21 +253,31 @@ def dataclass_to_dict(data: "dataclasses.dataclass") -> Dict: def match_and_substitute_params(mapping: Optional[Union[Callable, Dict]] = None, data: Optional[Any] = None) -> Any: r""" - 用来实现将输入：batch，或者输出：outputs，通过 `mapping` 将键值进行更换的功能； - 该函数应用于 `input_mapping` 和 `output_mapping`； - 对于 `input_mapping`，该函数会在 `TrainBatchLoop` 中取完数据后立刻被调用； - 对于 `output_mapping`，该函数会在 `Trainer.train_step` 以及 `Evaluator.train_step` 中得到结果后立刻被调用； + 用来实现将输入的 ``batch``，或者输出的 ``outputs``，通过 ``mapping`` 将键值进行更换的功能； + 该函数应用于 ``input_mapping`` 和 ``output_mapping``； - 转换的逻辑按优先级依次为： + 对于 ``input_mapping``，该函数会在 :class:`~fastNLP.core.controllers.TrainBatchLoop` 中取完数据后立刻被调用； + 对于 ``output_mapping``，该函数会在 :class:`~fastNLP.core.Trainer` 的 :meth:`~fastNLP.core.Trainer.train_step` + 以及 :class:`~fastNLP.core.Evaluator` 的 :meth:`~fastNLP.core.Evaluator.train_step` 中得到结果后立刻被调用； - 1. 如果 `mapping` 是一个函数，那么会直接返回 `mapping(data)`； - 2. 如果 `mapping` 是一个 `Dict`，那么 `data` 的类型只能为以下三种： [`Dict`, `dataclass`, `Sequence`]； - 如果 `data` 是 `Dict`，那么该函数会将 `data` 的 key 替换为 mapping[key]； - 如果 `data` 是 `dataclass`，那么该函数会先使用 `dataclasses.asdict` 函数将其转换为 `Dict`，然后进行转换； - 如果 `data` 是 `Sequence`，那么该函数会先将其转换成一个对应的 `Dict`：{"_0": list[0], "_1": list[1], ...}，然后使用 - mapping对这个 `Dict` 进行转换，如果没有匹配上mapping中的key则保持"_number"这个形式。 + 转换的逻辑按优先级依次为： - :param mapping: 用于转换的字典或者函数；mapping是函数时，返回值必须为字典类型。 + 1. 如果 ``mapping`` 是一个函数，那么会直接返回 ``mapping(data)``； + 2. 如果 ``mapping`` 是一个 ``Dict``，那么 ``data`` 的类型只能为以下三种： ``[Dict, dataclass, Sequence]``； + + * 如果 ``data`` 是 ``Dict``，那么该函数会将 ``data`` 的 ``key`` 替换为 ``mapping[key]``； + * 如果 ``data`` 是 ``dataclass``，那么该函数会先使用 :func:`dataclasses.asdict` 函数将其转换为 ``Dict``，然后进行转换； + * 如果 ``data`` 是 ``Sequence``，那么该函数会先将其转换成一个对应的字典:: + + { + "_0": list[0], + "_1": list[1], + ... + } + + 然后使用 ``mapping`` 对这个 ``Dict`` 进行转换，如果没有匹配上 ``mapping`` 中的 ``key`` 则保持 ``\'\_number\'`` 这个形式。 + + :param mapping: 用于转换的字典或者函数；``mapping`` 是函数时，返回值必须为字典类型。 :param data: 需要被转换的对象； :return: 返回转换好的结果； """ @@ -320,21 +330,20 @@ def apply_to_collection( include_none: bool = True, **kwargs: Any, ) -> Any: - """将函数 function 递归地在 data 中的元素执行，但是仅在满足元素为 dtype 时执行。 - - this function credit to: https://github.com/PyTorchLightning/pytorch-lightning - Args: - data: the collection to apply the function to - dtype: the given function will be applied to all elements of this dtype - function: the function to apply - *args: positional arguments (will be forwarded to calls of ``function``) - wrong_dtype: the given function won't be applied if this type is specified and the given collections - is of the ``wrong_dtype`` even if it is of type ``dtype`` - include_none: Whether to include an element if the output of ``function`` is ``None``. - **kwargs: keyword arguments (will be forwarded to calls of ``function``) - - Returns: - The resulting collection + """ + 使用函数 ``function`` 递归地在 ``data`` 中的元素执行，但是仅在满足元素为 ``dtype`` 时执行。 + + 该函数参考了 `pytorch-lightning `_ 的实现 + + :param data: 需要进行处理的数据集合或数据 + :param dtype: 数据的类型，函数 ``function`` 只会被应用于 ``data`` 中类型为 ``dtype`` 的数据 + :param function: 对数据进行处理的函数 + :param args: ``function`` 所需要的其它参数 + :param wrong_dtype: ``function`` 一定不会生效的数据类型。如果数据既是 ``wrong_dtype`` 类型又是 ``dtype`` 类型 + 那么也不会生效。 + :param include_none: 是否包含执行结果为 ``None`` 的数据，默认为 ``True``。 + :param kwargs: ``function`` 所需要的其它参数 + :return: 经过 ``function`` 处理后的数据集合 """ # Breaking condition if isinstance(data, dtype) and (wrong_dtype is None or not isinstance(data, wrong_dtype)): @@ -402,16 +411,18 @@ def apply_to_collection( @contextmanager def nullcontext(): r""" - 用来实现一个什么 dummy 的 context 上下文环境； + 实现一个什么都不做的上下文环境 """ yield def sub_column(string: str, c: int, c_size: int, title: str) -> str: r""" + 对传入的字符串进行截断，方便在命令行中显示 + :param string: 要被截断的字符串 :param c: 命令行列数 - :param c_size: instance或dataset field数 + :param c_size: :class:`~fastNLP.core.Instance` 或 :class:`fastNLP.core.DataSet` 的 ``field`` 数目 :param title: 列名 :return: 对一个过长的列进行截断的结果 """ @@ -442,18 +453,17 @@ def _is_iterable(value): def pretty_table_printer(dataset_or_ins) -> PrettyTable: r""" - :param dataset_or_ins: 传入一个dataSet或者instance - - .. code-block:: + 在 ``fastNLP`` 中展示数据的函数:: - ins = Instance(field_1=[1, 1, 1], field_2=[2, 2, 2], field_3=["a", "b", "c"]) + >>> ins = Instance(field_1=[1, 1, 1], field_2=[2, 2, 2], field_3=["a", "b", "c"]) +-----------+-----------+-----------------+ | field_1 | field_2 | field_3 | +-----------+-----------+-----------------+ | [1, 1, 1] | [2, 2, 2] | ['a', 'b', 'c'] | +-----------+-----------+-----------------+ - :return: 以 pretty table的形式返回根据terminal大小进行自动截断 + :param dataset_or_ins: 要展示的 :class:`~fastNLP.core.DataSet` 或者 :class:`~fastNLP.core.Instance` + :return: 根据 ``terminal`` 大小进行自动截断的数据表格 """ x = PrettyTable() try: @@ -486,7 +496,7 @@ def pretty_table_printer(dataset_or_ins) -> PrettyTable: class Option(dict): - r"""a dict can treat keys as attributes""" + r"""将键转化为属性的字典类型""" def __getattr__(self, item): try: @@ -516,11 +526,10 @@ _emitted_deprecation_warnings = set() def deprecated(help_message: Optional[str] = None): - """Decorator to mark a function as deprecated. + """ + 标记当前功能已经过时的装饰器。 - Args: - help_message (`Optional[str]`): An optional message to guide the user on how to - switch to non-deprecated usage of the library. + :param help_message: 一段指引信息，告知用户如何将代码切换为当前版本提倡的用法。 """ def decorator(deprecated_function: Callable): @@ -549,11 +558,10 @@ def deprecated(help_message: Optional[str] = None): return decorator -def seq_len_to_mask(seq_len, max_len=None): +def seq_len_to_mask(seq_len, max_len: Optional[int]): r""" - 将一个表示sequence length的一维数组转换为二维的mask，不包含的位置为0。 - 转变 1-d seq_len到2-d mask. + 将一个表示 ``sequence length`` 的一维数组转换为二维的 ``mask`` ，不包含的位置为 **0**。 .. code-block:: @@ -570,10 +578,11 @@ def seq_len_to_mask(seq_len, max_len=None): >>>print(mask.size()) torch.Size([14, 100]) - :param np.ndarray,torch.LongTensor seq_len: shape将是(B,) - :param int max_len: 将长度pad到这个长度。默认(None)使用的是seq_len中最长的长度。但在nn.DataParallel的场景下可能不同卡的seq_len会有 - 区别，所以需要传入一个max_len使得mask的长度是pad到该长度。 - :return: np.ndarray, torch.Tensor 。shape将是(B, max_length)， 元素类似为bool或torch.uint8 + :param seq_len: 大小为是 ``(B,)`` 的长度序列 + :param int max_len: 将长度 ``pad`` 到 ``max_len``。默认情况（为 ``None``）使用的是 ``seq_len`` 中最长的长度。 + 但在 :class:`torch.nn.DataParallel` 等分布式的场景下可能不同卡的 ``seq_len`` 会有区别，所以需要传入 + 一个 ``max_len`` 使得 ``mask`` 的长度 ``pad`` 到该长度。 + :return: 大小为 ``(B, max_len)`` 的 ``mask``， 元素类型为 ``bool`` 或 ``uint8`` """ if isinstance(seq_len, np.ndarray): assert len(np.shape(seq_len)) == 1, f"seq_len can only have one dimension, got {len(np.shape(seq_len))}." diff --git a/fastNLP/envs/utils.py b/fastNLP/envs/utils.py index 355c2448..3936771e 100644 --- a/fastNLP/envs/utils.py +++ b/fastNLP/envs/utils.py @@ -6,6 +6,7 @@ from packaging.version import Version import subprocess import pkg_resources +__all__ = [] def _module_available(module_path: str) -> bool: """Check if a path is available in your environment. @@ -48,10 +49,11 @@ def _compare_version(package: str, op: Callable, version: str, use_base_version: pkg_version = Version(pkg_version.base_version) return op(pkg_version, Version(version)) -def get_gpu_count(): +def get_gpu_count() -> int: """ - 利用命令行获取gpu数目的函数 - :return: gpu数目，如果没有显卡设备则为-1 + 利用命令行获取 ``gpu`` 数目的函数 + + :return: 显卡数目，如果没有显卡设备则为-1 """ try: lines = subprocess.check_output(['nvidia-smi', '--query-gpu=memory.used', '--format=csv']) From 2b9e09e07af5a40e02f6ce77b192391277c9f73b Mon Sep 17 00:00:00 2001 From: yh_cc Date: Tue, 10 May 2022 02:37:01 +0800 Subject: [PATCH 06/15] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dfilter=20state=20bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/core/callbacks/callback_manager.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/fastNLP/core/callbacks/callback_manager.py b/fastNLP/core/callbacks/callback_manager.py index f34c5dd3..82b1a756 100644 --- a/fastNLP/core/callbacks/callback_manager.py +++ b/fastNLP/core/callbacks/callback_manager.py @@ -188,6 +188,8 @@ class CallbackManager: for each_callback_filters in self._callback_filters: if each_callback_filters[0] not in _record_duplicated_callback_names: _record_duplicated_callback_names.add(each_callback_filters[0]) + if 'filter_states' not in states[each_callback_filters[0]]: + states[each_callback_filters[0]]["filter_states"] = {} states[each_callback_filters[0]]["filter_states"][each_callback_filters[1]] = each_callback_filters[2].state_dict() # 3. 保存 callback_counter； @@ -214,7 +216,9 @@ class CallbackManager: if each_callback_filters[0] in states: if each_callback_filters[0] not in _already_loaded_callback_names: _already_loaded_callback_names.add(each_callback_filters[0]) - each_callback_filters[2].load_state_dict(states[each_callback_filters[0]]["filter_states"][each_callback_filters[1]]) + if 'filter_states' in states[each_callback_filters[0]] and \ + each_callback_filters[1] in states[each_callback_filters[0]]['filter_states']: + each_callback_filters[2].load_state_dict(states[each_callback_filters[0]]['filter_states'][each_callback_filters[1]]) else: _duplicated_callback_names.add(each_callback_filters[0]) From ef892a7aed4eb1bd622155113d22789cd9209773 Mon Sep 17 00:00:00 2001 From: x54-729 <17307130121@fudan.edu.cn> Date: Mon, 9 May 2022 19:38:01 +0000 Subject: [PATCH 07/15] =?UTF-8?q?1.=20=E6=94=AF=E6=8C=81=E5=9C=A8=E4=B8=8D?= =?UTF-8?q?=E8=AE=BE=E7=BD=AEbackend=E7=9A=84=E6=83=85=E5=86=B5=E4=B8=8B?= =?UTF-8?q?=E8=BF=90=E8=A1=8C=E5=8D=95=E5=8D=A1=E7=9A=84paddle=E7=A8=8B?= =?UTF-8?q?=E5=BA=8F=202.=E5=BD=93=E9=80=9A=E8=BF=87launch=E5=90=AF?= =?UTF-8?q?=E5=8A=A8=E4=B8=94=E9=99=90=E5=88=B6=E6=98=BE=E5=8D=A1=E6=97=B6?= =?UTF-8?q?=E7=9A=84paddle=E5=A4=9A=E5=8D=A1=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/core/drivers/paddle_driver/fleet.py | 21 +- .../drivers/paddle_driver/fleet_launcher.py | 7 +- .../paddle_driver/initialize_paddle_driver.py | 22 +- .../drivers/paddle_driver/paddle_driver.py | 7 +- .../drivers/paddle_driver/single_device.py | 17 +- .../metrics/backend/paddle_backend/backend.py | 5 +- fastNLP/core/utils/paddle_utils.py | 42 ++-- fastNLP/envs/set_backend.py | 18 +- tests/core/controllers/_test_trainer_fleet.py | 19 +- .../_test_trainer_fleet_outside.py | 17 +- .../core/controllers/_test_trainer_jittor.py | 237 ++++++++++++++++++ tests/core/controllers/imdb.py | 110 ++++++++ tests/core/controllers/test_trainer_paddle.py | 5 + tests/core/utils/test_paddle_utils.py | 33 +-- 14 files changed, 477 insertions(+), 83 deletions(-) create mode 100644 tests/core/controllers/_test_trainer_jittor.py create mode 100644 tests/core/controllers/imdb.py diff --git a/fastNLP/core/drivers/paddle_driver/fleet.py b/fastNLP/core/drivers/paddle_driver/fleet.py index e5b2a06f..d09cacc1 100644 --- a/fastNLP/core/drivers/paddle_driver/fleet.py +++ b/fastNLP/core/drivers/paddle_driver/fleet.py @@ -19,6 +19,7 @@ from fastNLP.core.utils import ( check_user_specific_params, is_in_paddle_dist, is_in_paddle_dist, + get_paddle_device_id, ) from fastNLP.envs.distributed import rank_zero_rm from fastNLP.core.samplers import ( @@ -31,7 +32,12 @@ from fastNLP.core.samplers import ( re_instantiate_sampler, conversion_between_reproducible_and_unrepeated_sampler, ) -from fastNLP.envs.env import FASTNLP_DISTRIBUTED_CHECK, FASTNLP_GLOBAL_SEED, FASTNLP_NO_SYNC +from fastNLP.envs.env import ( + FASTNLP_DISTRIBUTED_CHECK, + FASTNLP_GLOBAL_SEED, + FASTNLP_NO_SYNC, + USER_CUDA_VISIBLE_DEVICES, +) from fastNLP.core.log import logger if _NEED_IMPORT_PADDLE: @@ -51,7 +57,7 @@ class PaddleFleetDriver(PaddleDriver): def __init__( self, model, - parallel_device: Optional[Union[List[int], int]], + parallel_device: Optional[Union[List[str], str]], is_pull_by_paddle_run: bool = False, fp16: bool = False, **kwargs @@ -185,6 +191,8 @@ class PaddleFleetDriver(PaddleDriver): 不管是什么情况，`PaddleFleetDriver` 在 `setup` 函数的最后，都会将所有进程的 pid 主动记录下来，这样当一个进程出现 exception 后， driver 的 on_exception 函数就会被 trainer 调用，其会调用 os.kill 指令将其它进程 kill 掉； """ + # if USER_CUDA_VISIBLE_DEVICES not in os.environ: + # raise RuntimeError("To run paddle distributed training, please set `FASTNLP_BACKEND` to 'paddle' before using FastNLP.") super(PaddleFleetDriver, self).__init__(model, fp16=fp16, **kwargs) # 如果不是通过 launch 启动，要求用户必须传入 parallel_device @@ -229,9 +237,9 @@ class PaddleFleetDriver(PaddleDriver): self._data_device = f"gpu:{self._data_device}" elif not isinstance(self._data_device, str): raise ValueError("Parameter `device` is wrong type, please check our documentation for the right use.") - if self.outside_fleet and paddle.device.get_device() != self._data_device: - logger.warning("`Parameter data_device` is not equal to paddle.deivce.get_device(), " - "please keep them equal to avoid some potential bugs.") + # if self.outside_fleet and paddle.device.get_device() != self._data_device: + # logger.warning("`Parameter data_device` is not equal to paddle.deivce.get_device(), " + # "please keep them equal to avoid some potential bugs.") self.world_size = None self.global_rank = 0 @@ -304,7 +312,8 @@ class PaddleFleetDriver(PaddleDriver): else: # 已经设置过一次，保证参数必须是一样的 pre_gpus = os.environ[FASTNLP_DISTRIBUTED_CHECK] - pre_gpus = [int (x) for x in pre_gpus.split(",")] + pre_gpus = [int(x) for x in pre_gpus.split(",")] + cur_gpus = [get_paddle_device_id(g) for g in self.parallel_device] if sorted(pre_gpus) != sorted(self.parallel_device): raise RuntimeError("Notice you are using `PaddleFleetDriver` after one instantiated `PaddleFleetDriver`, it is not" "allowed that your second `PaddleFleetDriver` has a new setting of parameters `parallel_device`.") diff --git a/fastNLP/core/drivers/paddle_driver/fleet_launcher.py b/fastNLP/core/drivers/paddle_driver/fleet_launcher.py index ca341db5..b53680cc 100644 --- a/fastNLP/core/drivers/paddle_driver/fleet_launcher.py +++ b/fastNLP/core/drivers/paddle_driver/fleet_launcher.py @@ -11,11 +11,14 @@ from fastNLP.envs.env import ( FASTNLP_LOG_LEVEL, FASTNLP_GLOBAL_SEED, ) +from fastNLP.core.utils import get_paddle_device_id from .utils import ( find_free_ports, reset_seed, ) +__all__ = [] + # 记录各个进程信息 class SubTrainer(object): """ @@ -34,11 +37,11 @@ class FleetLauncher: """ def __init__( self, - devices: List[int], + devices: List[str], output_from_new_proc: str = "only_error" ): - self.devices = devices + self.devices = [ get_paddle_device_id(g) for g in devices] self.output_from_new_proc = output_from_new_proc self.setup() diff --git a/fastNLP/core/drivers/paddle_driver/initialize_paddle_driver.py b/fastNLP/core/drivers/paddle_driver/initialize_paddle_driver.py index 60e8afc0..aa1b2db5 100644 --- a/fastNLP/core/drivers/paddle_driver/initialize_paddle_driver.py +++ b/fastNLP/core/drivers/paddle_driver/initialize_paddle_driver.py @@ -7,7 +7,7 @@ from .single_device import PaddleSingleDriver from .fleet import PaddleFleetDriver from fastNLP.envs.imports import _NEED_IMPORT_PADDLE -from fastNLP.core.utils import is_in_paddle_launch_dist +from fastNLP.core.utils import is_in_paddle_launch_dist, get_paddle_gpu_str from fastNLP.core.log import logger if _NEED_IMPORT_PADDLE: @@ -30,27 +30,28 @@ def initialize_paddle_driver(driver: str, device: Optional[Union[str, int, List[ """ if driver != "paddle": raise ValueError("When initialize PaddleDriver, parameter `driver` must be 'paddle'.") + user_visible_devices = os.getenv("USER_CUDA_VISIBLE_DEVICES") if is_in_paddle_launch_dist(): if device is not None: logger.warning_once("Parameter `device` would be ignored when you are using `paddle.distributed.launch` to pull " - "up your script. And we will directly get the local device via " - "and `os.environ['CUDA_VISIBLE_DEVICES']``.") - device = [int(g) for g in os.environ["CUDA_VISIBLE_DEVICES"].split(",")] - # TODO 目前一个进程仅对应一个卡，所以暂时传入一个 int + "up your script. And we will directly get the local device via environment variables.") + _visible_list = user_visible_devices.split(",") + device = [ f"gpu:{_visible_list.index(g) }" for g in os.environ["CUDA_VISIBLE_DEVICES"].split(",")] + # TODO 目前一个进程仅对应一个卡，所以暂时传入单个 return PaddleFleetDriver(model, device[0], True, **kwargs) - user_visible_devices = os.getenv("USER_CUDA_VISIBLE_DEVICES") if user_visible_devices is None: - raise RuntimeError("`USER_CUDA_VISIBLE_DEVICES` cannot be None, please check if you have set " - "`FASTNLP_BACKEND` to 'paddle' before using FastNLP.") - _could_use_device_num = len(user_visible_devices.split(",")) + _could_use_device_num = paddle.device.cuda.device_count() + else: + _could_use_device_num = len(user_visible_devices.split(",")) + if isinstance(device, int): if device < 0 and device != -1: raise ValueError("Parameter `device` can only be '-1' when it is smaller than 0.") if device >= _could_use_device_num: raise ValueError("The gpu device that parameter `device` specifies is not existed.") if device == -1: - device = list(range(_could_use_device_num)) + device = [ get_paddle_gpu_str(g) for g in range(_could_use_device_num)] elif isinstance(device, Sequence) and not isinstance(device, str): device = list(set(device)) for each in device: @@ -61,6 +62,7 @@ def initialize_paddle_driver(driver: str, device: Optional[Union[str, int, List[ elif each >= _could_use_device_num: raise ValueError("When parameter `device` is 'Sequence' type, the value in it should not be bigger than" " the available gpu number.") + device = [get_paddle_gpu_str(g) for g in device] elif device is not None and not isinstance(device, str): raise ValueError("Parameter `device` is wrong type, please check our documentation for the right use.") if isinstance(device, List): diff --git a/fastNLP/core/drivers/paddle_driver/paddle_driver.py b/fastNLP/core/drivers/paddle_driver/paddle_driver.py index 00b0da4e..cf35af3a 100644 --- a/fastNLP/core/drivers/paddle_driver/paddle_driver.py +++ b/fastNLP/core/drivers/paddle_driver/paddle_driver.py @@ -7,6 +7,8 @@ from dataclasses import dataclass import numpy as np +from fastNLP.envs.env import USER_CUDA_VISIBLE_DEVICES + from .utils import _build_fp16_env, optimizer_state_to_device, DummyGradScaler from fastNLP.envs.imports import _NEED_IMPORT_PADDLE from fastNLP.core.drivers.driver import Driver @@ -369,7 +371,10 @@ class PaddleDriver(Driver): :return: 将移动到指定机器上的 batch 对象返回； """ - device = get_device_from_visible(self.data_device) + if USER_CUDA_VISIBLE_DEVICES in os.environ: + device = get_device_from_visible(self.data_device) + else: + device = self.data_device return paddle_move_data_to_device(batch, device) @staticmethod diff --git a/fastNLP/core/drivers/paddle_driver/single_device.py b/fastNLP/core/drivers/paddle_driver/single_device.py index 69b58954..6d553fea 100644 --- a/fastNLP/core/drivers/paddle_driver/single_device.py +++ b/fastNLP/core/drivers/paddle_driver/single_device.py @@ -40,9 +40,6 @@ class PaddleSingleDriver(PaddleDriver): raise ValueError("`paddle.DataParallel` is not supported in `PaddleSingleDriver`") cuda_visible_devices = os.getenv(USER_CUDA_VISIBLE_DEVICES) - if cuda_visible_devices is None: - raise RuntimeError("`USER_CUDA_VISIBLE_DEVICES` cannot be None, please check if you have set " - "`FASTNLP_BACKEND` to 'paddle' before using FastNLP.") if cuda_visible_devices == "": device = "cpu" logger.info("You have set `CUDA_VISIBLE_DEVICES` to '' in system environment variable, and we are gonna to" @@ -54,11 +51,9 @@ class PaddleSingleDriver(PaddleDriver): raise ValueError("Parameter `device` can not be None in `PaddleSingleDriver`.") if device != "cpu": - if isinstance(device, int): - device_id = device - else: - device_id = get_paddle_device_id(device) - os.environ["CUDA_VISIBLE_DEVICES"] = cuda_visible_devices.split(",")[device_id] + device_id = get_paddle_device_id(device) + if cuda_visible_devices is not None: + os.environ["CUDA_VISIBLE_DEVICES"] = cuda_visible_devices.split(",")[device_id] self.model_device = get_paddle_gpu_str(device) self.local_rank = 0 @@ -69,7 +64,11 @@ class PaddleSingleDriver(PaddleDriver): r""" 该函数用来初始化训练环境，用于设置当前训练的设备，并将模型迁移到对应设备上。 """ - device = get_device_from_visible(self.model_device, output_type=str) + if USER_CUDA_VISIBLE_DEVICES in os.environ: + device = get_device_from_visible(self.data_device) + else: + device = self.data_device + paddle.device.set_device(device) with contextlib.redirect_stdout(None): self.model.to(device) diff --git a/fastNLP/core/metrics/backend/paddle_backend/backend.py b/fastNLP/core/metrics/backend/paddle_backend/backend.py index aa57bbc2..74cf6b82 100644 --- a/fastNLP/core/metrics/backend/paddle_backend/backend.py +++ b/fastNLP/core/metrics/backend/paddle_backend/backend.py @@ -1,3 +1,4 @@ +import os from typing import List, Any import numpy as np @@ -7,6 +8,7 @@ from fastNLP.core.utils.paddle_utils import paddle_to, get_device_from_visible from fastNLP.core.metrics.utils import AggregateMethodError from fastNLP.core.drivers.paddle_driver.dist_utils import fastnlp_paddle_all_gather from fastNLP.envs.imports import _NEED_IMPORT_PADDLE +from fastNLP.envs.env import USER_CUDA_VISIBLE_DEVICES if _NEED_IMPORT_PADDLE: import paddle @@ -79,7 +81,8 @@ class PaddleBackend(Backend): raise ValueError(f"tensor: {tensor} can not convert to ndarray!") def move_tensor_to_device(self, tensor, device): - device = get_device_from_visible(device) + if USER_CUDA_VISIBLE_DEVICES in os.environ: + device = get_device_from_visible(device) return paddle_to(tensor, device) def all_gather_object(self, obj, group=None) -> List: diff --git a/fastNLP/core/utils/paddle_utils.py b/fastNLP/core/utils/paddle_utils.py index db68879f..13fe7b07 100644 --- a/fastNLP/core/utils/paddle_utils.py +++ b/fastNLP/core/utils/paddle_utils.py @@ -21,38 +21,32 @@ if _NEED_IMPORT_PADDLE: from .utils import apply_to_collection -def get_device_from_visible(device: Union[str, int], output_type=int): +def get_device_from_visible(device: Union[str, int]) -> str: """ - 在有 CUDA_VISIBLE_DEVICES 的情况下，获取对应的设备。 + 在有 ``CUDA_VISIBLE_DEVICES`` 的情况下，获取对应的设备。 如 CUDA_VISIBLE_DEVICES=2,3 ，device=3 ，则返回1。 :param device: 未转化的设备名 - :param output_type: 返回值的类型 - :return: 转化后的设备id + :return: 转化后的设备，格式为 ``gpu:x`` """ - if output_type not in [int, str]: - raise ValueError("Parameter `output_type` should be one of these types: [int, str]") if device == "cpu": return device cuda_visible_devices = os.getenv("CUDA_VISIBLE_DEVICES") user_visible_devices = os.getenv(USER_CUDA_VISIBLE_DEVICES) - if user_visible_devices is None: - raise RuntimeError("`USER_CUDA_VISIBLE_DEVICES` cannot be None, please check if you have set " - "`FASTNLP_BACKEND` to 'paddle' before using FastNLP.") - idx = get_paddle_device_id(device) - # 利用 USER_CUDA_VISIBLDE_DEVICES 获取用户期望的设备 - if user_visible_devices is None: - raise RuntimeError("This situation cannot happen, please report a bug to us.") - idx = user_visible_devices.split(",")[idx] - - cuda_visible_devices_list = cuda_visible_devices.split(',') - if idx not in cuda_visible_devices_list: - raise ValueError(f"Can't find your devices {idx} in CUDA_VISIBLE_DEVICES[{cuda_visible_devices}]. ") - res = cuda_visible_devices_list.index(idx) - if output_type == int: - return res + if cuda_visible_devices is not None: + idx = get_paddle_device_id(device) + if user_visible_devices is not None: + # 此时一定发生在分布式的情况下，利用 USER_CUDA_VISIBLDE_DEVICES 获取用户期望的设备 + idx = user_visible_devices.split(",")[idx] + else: + idx = str(idx) + + cuda_visible_devices_list = cuda_visible_devices.split(',') + if idx not in cuda_visible_devices_list: + raise ValueError(f"Can't find your devices {idx} in CUDA_VISIBLE_DEVICES[{cuda_visible_devices}]. ") + return f"gpu:{cuda_visible_devices_list.index(idx)}" else: - return f"gpu:{res}" + return get_paddle_gpu_str(device) def paddle_to(data, device: Union[str, int]): """ @@ -70,7 +64,7 @@ def paddle_to(data, device: Union[str, int]): return data.cuda(get_paddle_device_id(device)) -def get_paddle_gpu_str(device: Union[str, int]): +def get_paddle_gpu_str(device: Union[str, int]) -> str: """ 获得 `gpu:x` 类型的设备名 @@ -82,7 +76,7 @@ def get_paddle_gpu_str(device: Union[str, int]): return f"gpu:{device}" -def get_paddle_device_id(device: Union[str, int]): +def get_paddle_device_id(device: Union[str, int]) -> int: """ 获得 gpu 的设备id diff --git a/fastNLP/envs/set_backend.py b/fastNLP/envs/set_backend.py index b75a9610..e6b9bf59 100644 --- a/fastNLP/envs/set_backend.py +++ b/fastNLP/envs/set_backend.py @@ -51,23 +51,33 @@ def _set_backend(): assert _module_available(backend), f"You must have {backend} available to use {backend} backend." assert 'paddle' not in sys.modules, "You have to use `set_backend()` before `import paddle`." user_visible_devices = os.getenv(USER_CUDA_VISIBLE_DEVICES) + cuda_visible_devices = os.getenv("CUDA_VISIBLE_DEVICES") if 'PADDLE_RANK_IN_NODE' in os.environ and 'FLAGS_selected_gpus' in os.environ: # 在分布式子进程下，根据 USER_VISIBLE_DEVICES 得到进程真正占有的设备 selected_gpus = os.environ['FLAGS_selected_gpus'].split(',') if user_visible_devices is not None: - # 用户通过 CUDA_VISIBLE_DEVICES 启动了分布式训练 + # 用户使用 fastNLP 启动了分布式训练 # 此时经过 set_backend，用户的设置会保存在 USER_CUDA_VISIBLE_DEVICES 中 - # 我们需要从中找到真正使用的设备编号 + # 我们需要从中转换为用户找到真正使用的设备编号 user_visible_devices = user_visible_devices.split(",") - selected_gpus = ",".join([user_visible_devices[int(i)] for i in selected_gpus]) + selected_gpus = [user_visible_devices[int(i)] for i in selected_gpus] + # 没有找到 USER_CUDA_VISIBLE_DEVICES，说明用户是直接用 launch 启动的 + elif cuda_visible_devices: + # 用户设置了可见设备，需要进行转换 + # 如 CUDA_VISIBLE_DEVICES = 0,2,3 --gpus=0,2,3 + # 在 rank1 中此时 selected_gpus = ['1']，需要转换为设备 2 + os.environ[USER_CUDA_VISIBLE_DEVICES] = cuda_visible_devices + cuda_visible_devices = cuda_visible_devices.split(",") + selected_gpus = [cuda_visible_devices[int(i)] for i in selected_gpus] else: - # 没有找到 USER_CUDA_VISIBLE_DEVICES，则将之设置为所有的设备 + # 用户没有设置可见设备，则赋值成所有的设备 os.environ[USER_CUDA_VISIBLE_DEVICES] = ",".join(map(str, list( range(get_gpu_count()) ))) os.environ['CUDA_VISIBLE_DEVICES'] = ",".join(selected_gpus) os.environ['FLAGS_selected_gpus'] = ",".join([str(g) for g in range(len(selected_gpus))]) os.environ['FLAGS_selected_accelerators'] = ",".join([str(g) for g in range(len(selected_gpus))]) + elif 'CUDA_VISIBLE_DEVICES' in os.environ: # 主进程中，用户设置了 CUDA_VISIBLE_DEVICES # 将用户设置的 CUDA_VISIBLE_DEVICES hack 掉 diff --git a/tests/core/controllers/_test_trainer_fleet.py b/tests/core/controllers/_test_trainer_fleet.py index 1a01bb5d..dd87f348 100644 --- a/tests/core/controllers/_test_trainer_fleet.py +++ b/tests/core/controllers/_test_trainer_fleet.py @@ -1,7 +1,15 @@ """ -这个文件测试用户以python -m paddle.distributed.launch 启动的情况 -看看有没有用pytest执行的机会 -FASTNLP_BACKEND=paddle python -m paddle.distributed.launch --gpus=0,2,3 _test_trainer_fleet.py +这个文件测试多卡情况下使用 paddle 的情况:: + + >>> # 测试用 python -m paddle.distributed.launch 启动 + >>> FASTNLP_BACKEND=paddle python -m paddle.distributed.launch --gpus=0,2,3 _test_trainer_fleet.py + >>> # 测试在限制 GPU 的情况下用 python -m paddle.distributed.launch 启动 + >>> CUDA_VISIBLE_DEVICES=0,2,3 FASTNLP_BACKEND=paddle python -m paddle.distributed.launch --gpus=0,2,3 _test_trainer_fleet.py + >>> # 测试直接使用多卡 + >>> FASTNLP_BACKEND=paddle python _test_trainer_fleet.py + >>> # 测试在限制 GPU 的情况下直接使用多卡 + >>> CUDA_VISIBLE_DEVICES=3,4,5,6 FASTNLP_BACKEND=paddle python _test_trainer_fleet.py + """ import os import sys @@ -71,14 +79,13 @@ def test_trainer_fleet( n_epochs=n_epochs, callbacks=callbacks, - output_from_new_proc="logs", + # output_from_new_proc="logs", ) trainer.run() if __name__ == "__main__": driver = "paddle" - device = [0,2,3] - # driver = "paddle" + device = [0,1,3] # device = 2 callbacks = [ # RecordMetricCallback(monitor="acc#acc", metric_threshold=0.0, larger_better=True), diff --git a/tests/core/controllers/_test_trainer_fleet_outside.py b/tests/core/controllers/_test_trainer_fleet_outside.py index 1ab2e624..f5b7fc4d 100644 --- a/tests/core/controllers/_test_trainer_fleet_outside.py +++ b/tests/core/controllers/_test_trainer_fleet_outside.py @@ -1,7 +1,11 @@ """ -这个文件测试用户以python -m paddle.distributed.launch 启动的情况 -并且自己初始化了 fleet -FASTNLP_BACKEND=paddle python -m paddle.distributed.launch --gpus=0,2,3 _test_trainer_fleet_outside.py +这个文件测试用户自己初始化分布式环境后使用 paddle 的情况: + + >>> # 测试用 python -m paddle.distributed.launch 启动 + >>> FASTNLP_BACKEND=paddle python -m paddle.distributed.launch --gpus=0,2,3 _test_trainer_fleet_outside.py + >>> # 测试在限制 GPU 的情况下用 python -m paddle.distributed.launch 启动 + >>> CUDA_VISIBLE_DEVICES=0,2,3 FASTNLP_BACKEND=paddle python -m paddle.distributed.launch --gpus=0,2,3 _test_trainer_fleet_outside.py + """ import os import sys @@ -63,6 +67,7 @@ def test_trainer_fleet( validate_dataloaders = val_dataloader validate_every = MNISTTrainFleetConfig.validate_every metrics = {"acc": Accuracy()} + data_device = f'gpu:{os.environ["USER_CUDA_VISIBLE_DEVICES"].split(",").index(os.environ["CUDA_VISIBLE_DEVICES"])}' trainer = Trainer( model=model, driver=driver, @@ -77,14 +82,14 @@ def test_trainer_fleet( n_epochs=n_epochs, callbacks=callbacks, - output_from_new_proc="logs", - data_device=f"gpu:{os.environ['CUDA_VISIBLE_DEVICES']}" + # output_from_new_proc="logs", + data_device=data_device ) trainer.run() if __name__ == "__main__": driver = "paddle" - device = [0,2,3] + device = [0,1,3] callbacks = [ # RecordMetricCallback(monitor="acc#acc", metric_threshold=0.0, larger_better=True), RichCallback(5), diff --git a/tests/core/controllers/_test_trainer_jittor.py b/tests/core/controllers/_test_trainer_jittor.py new file mode 100644 index 00000000..bc4b05f0 --- /dev/null +++ b/tests/core/controllers/_test_trainer_jittor.py @@ -0,0 +1,237 @@ +import os +import sys +import time +# os.environ["cuda_archs"] = "61" +# os.environ["FAS"] +os.environ["log_silent"] = "1" +sys.path.append("../../../") + +from datasets import load_dataset +from datasets import DatasetDict +import jittor as jt +from jittor import nn, Module +from jittor.dataset import Dataset +jt.flags.use_cuda = True + +from fastNLP.core.controllers.trainer import Trainer +from fastNLP.core.metrics.accuracy import Accuracy +from fastNLP.core.vocabulary import Vocabulary +from fastNLP.core.callbacks.progress_callback import RichCallback +from fastNLP.core.callbacks.callback import Callback +from fastNLP.core.dataloaders.jittor_dataloader.fdl import JittorDataLoader + +class TextClassificationDataset(Dataset): + def __init__(self, dataset): + super(TextClassificationDataset, self).__init__() + self.dataset = dataset + self.set_attrs(total_len=len(dataset)) + + def __getitem__(self, idx): + return {"x": self.dataset["input_ids"][idx], "y": self.dataset["label"][idx]} + + +class LSTM(Module): + + def __init__(self, num_of_words, hidden_size, features): + + self.embedding = nn.Embedding(num_of_words, features) + self.lstm = nn.LSTM(features, hidden_size, batch_first=True) + self.layer = nn.Linear(hidden_size, 2) + self.softmax = nn.Softmax(dim=1) + self.loss_fn = nn.CrossEntropyLoss() + + self.hidden_size = hidden_size + self.features = features + + def init_hidden(self, x): + # batch_first + batch_size = x.shape[0] + h0 = jt.randn(1, batch_size, hidden_size) + c0 = jt.randn(1, batch_size, hidden_size) + + return h0, c0 + + def execute(self, input_ids): + + output = self.embedding(input_ids) + # TODO 去除padding + output, (h, c) = self.lstm(output, self.init_hidden(output)) + # len, batch, hidden_size + output = self.layer(output[-1]) + + return output + + def train_step(self, x, y): + x = self(x) + outputs = self.loss_fn(x, y) + return {"loss": outputs} + + def evaluate_step(self, x, y): + x = self(x) + return {"pred": x, "target": y.reshape((-1,))} + + +class PrintWhileTrainingCallBack(Callback): + """ + 通过该Callback实现训练过程中loss的输出 + """ + + def __init__(self, print_every_epoch, print_every_batch): + self.print_every_epoch = print_every_epoch + self.print_every_batch = print_every_batch + + self.loss = 0 + self.start = 0 + self.epoch_start = 0 + + def on_train_begin(self, trainer): + """ + 在训练开始前输出信息 + """ + print("Start training. Total {} epochs and {} batches in each epoch.".format( + trainer.n_epochs, trainer.num_batches_per_epoch + )) + self.start = time.time() + + def on_before_backward(self, trainer, outputs): + """ + 每次反向传播前统计loss，用于计算平均值 + """ + loss = trainer.extract_loss_from_outputs(outputs) + loss = trainer.driver.tensor_to_numeric(loss) + self.loss += loss + + def on_train_epoch_begin(self, trainer): + self.epoch_start = time.time() + + def on_train_epoch_end(self, trainer): + """ + 在每经过一定epoch或最后一个epoch时输出当前epoch的平均loss和使用时间 + """ + if trainer.cur_epoch_idx % self.print_every_epoch == 0 \ + or trainer.cur_epoch_idx == trainer.n_epochs: + print("Epoch: {} Loss: {} Current epoch training time: {}s".format( + trainer.cur_epoch_idx, self.loss / trainer.num_batches_per_epoch, time.time() - self.epoch_start + )) + # 将loss清零 + self.loss = 0 + + def on_train_batch_end(self, trainer): + """ + 在每经过一定batch或最后一个batch时输出当前epoch截止目前的平均loss + """ + if trainer.batch_idx_in_epoch % self.print_every_batch == 0 \ + or trainer.batch_idx_in_epoch == trainer.num_batches_per_epoch: + print("\tBatch: {} Loss: {}".format( + trainer.batch_idx_in_epoch, self.loss / trainer.batch_idx_in_epoch + )) + + def on_train_end(self, trainer): + print("Total training time: {}s".format(time.time() - self.start)) + + +def process_data(ds: DatasetDict, vocabulary: Vocabulary, max_len=256) -> DatasetDict: + # 分词 + ds = ds.map(lambda x: {"input_ids": text_to_id(vocabulary, x["text"], max_len)}) + ds.set_format(type="numpy", columns=ds.column_names) + return ds + +def set_vocabulary(vocab, dataset): + + for data in dataset: + vocab.update(data["text"].split()) + return vocab + +def text_to_id(vocab, text: str, max_len): + text = text.split() + # to index + ids = [vocab.to_index(word) for word in text] + # padding + ids += [vocab.padding_idx] * (max_len - len(text)) + return ids[:max_len] + +def get_dataset(name, max_len, train_format="", test_format=""): + + # datasets + train_dataset = load_dataset(name, split="train" + train_format).shuffle(seed=123) + test_dataset = load_dataset(name, split="test" + test_format).shuffle(seed=321) + split = train_dataset.train_test_split(test_size=0.2, seed=123) + train_dataset = split["train"] + val_dataset = split["test"] + + vocab = Vocabulary() + vocab = set_vocabulary(vocab, train_dataset) + vocab = set_vocabulary(vocab, val_dataset) + + train_dataset = process_data(train_dataset, vocab, max_len) + val_dataset = process_data(val_dataset, vocab, max_len) + test_dataset = process_data(test_dataset, vocab, max_len) + + return TextClassificationDataset(train_dataset), TextClassificationDataset(val_dataset), \ + TextClassificationDataset(test_dataset), vocab + +if __name__ == "__main__": + + # 训练参数 + max_len = 20 + epochs = 40 + lr = 1 + batch_size = 64 + + features = 100 + hidden_size = 128 + + # 获取数据集 + # imdb.py SetFit/sst2 + train_data, val_data, test_data, vocab = get_dataset("SetFit/sst2", max_len, "", "") + # 使用dataloader + train_dataloader = JittorDataLoader( + dataset=train_data, + batch_size=batch_size, + shuffle=True, + num_workers=4, + ) + val_dataloader = JittorDataLoader( + dataset=val_data, + batch_size=batch_size, + shuffle=True, + num_workers=4, + ) + test_dataloader = JittorDataLoader( + dataset=test_data, + batch_size=1, + shuffle=False, + ) + + # 初始化模型 + model = LSTM(len(vocab), hidden_size, features) + + # 优化器 + # 也可以是多个优化器的list + optimizer = nn.SGD(model.parameters(), lr) + + # Metrics + metrics = {"acc": Accuracy()} + + # callbacks + callbacks = [ + PrintWhileTrainingCallBack(print_every_epoch=1, print_every_batch=10), + # RichCallback(), # print_every参数默认为1，即每一个batch更新一次进度条 + ] + + trainer = Trainer( + model=model, + driver="jittor", + device=[0,1,2,3,4], + optimizers=optimizer, + train_dataloader=train_dataloader, + validate_dataloaders=val_dataloader, + validate_every=-1, + input_mapping=None, + output_mapping=None, + metrics=metrics, + n_epochs=epochs, + callbacks=callbacks, + # progress_bar="raw" + ) + trainer.run() \ No newline at end of file diff --git a/tests/core/controllers/imdb.py b/tests/core/controllers/imdb.py new file mode 100644 index 00000000..cdf59047 --- /dev/null +++ b/tests/core/controllers/imdb.py @@ -0,0 +1,110 @@ +# coding=utf-8 +# Copyright 2020 The TensorFlow Datasets Authors and the HuggingFace Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""IMDB movie reviews dataset.""" + +import datasets +from datasets.tasks import TextClassification + + +_DESCRIPTION = """\ +Large Movie Review Dataset. +This is a dataset for binary sentiment classification containing substantially \ +more data than previous benchmark datasets. We provide a set of 25,000 highly \ +polar movie reviews for training, and 25,000 for testing. There is additional \ +unlabeled data for use as well.\ +""" + +_CITATION = """\ +@InProceedings{maas-EtAl:2011:ACL-HLT2011, + author = {Maas, Andrew L. and Daly, Raymond E. and Pham, Peter T. and Huang, Dan and Ng, Andrew Y. and Potts, Christopher}, + title = {Learning Word Vectors for Sentiment Analysis}, + booktitle = {Proceedings of the 49th Annual Meeting of the Association for Computational Linguistics: Human Language Technologies}, + month = {June}, + year = {2011}, + address = {Portland, Oregon, USA}, + publisher = {Association for Computational Linguistics}, + pages = {142--150}, + url = {http://www.aclweb.org/anthology/P11-1015} +} +""" + +_DOWNLOAD_URL = "http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz" + + +class IMDBReviewsConfig(datasets.BuilderConfig): + """BuilderConfig for IMDBReviews.""" + + def __init__(self, **kwargs): + """BuilderConfig for IMDBReviews. + Args: + **kwargs: keyword arguments forwarded to super. + """ + super(IMDBReviewsConfig, self).__init__(version=datasets.Version("1.0.0", ""), **kwargs) + + +class Imdb(datasets.GeneratorBasedBuilder): + """IMDB movie reviews dataset.""" + + BUILDER_CONFIGS = [ + IMDBReviewsConfig( + name="plain_text", + description="Plain text", + ) + ] + + def _info(self): + return datasets.DatasetInfo( + description=_DESCRIPTION, + features=datasets.Features( + {"text": datasets.Value("string"), "label": datasets.features.ClassLabel(names=["neg", "pos"])} + ), + supervised_keys=None, + homepage="http://ai.stanford.edu/~amaas/data/sentiment/", + citation=_CITATION, + task_templates=[TextClassification(text_column="text", label_column="label")], + ) + + def _split_generators(self, dl_manager): + archive = dl_manager.download(_DOWNLOAD_URL) + return [ + datasets.SplitGenerator( + name=datasets.Split.TRAIN, gen_kwargs={"files": dl_manager.iter_archive(archive), "split": "train"} + ), + datasets.SplitGenerator( + name=datasets.Split.TEST, gen_kwargs={"files": dl_manager.iter_archive(archive), "split": "test"} + ), + datasets.SplitGenerator( + name=datasets.Split("unsupervised"), + gen_kwargs={"files": dl_manager.iter_archive(archive), "split": "train", "labeled": False}, + ), + ] + + def _generate_examples(self, files, split, labeled=True): + """Generate aclImdb examples.""" + # For labeled examples, extract the label from the path. + if labeled: + label_mapping = {"pos": 1, "neg": 0} + for path, f in files: + if path.startswith(f"aclImdb/{split}"): + label = label_mapping.get(path.split("/")[2]) + if label is not None: + yield path, {"text": f.read().decode("utf-8"), "label": label} + else: + for path, f in files: + if path.startswith(f"aclImdb/{split}"): + if path.split("/")[2] == "unsup": + yield path, {"text": f.read().decode("utf-8"), "label": -1} \ No newline at end of file diff --git a/tests/core/controllers/test_trainer_paddle.py b/tests/core/controllers/test_trainer_paddle.py index d7bfaeaf..7945e2c6 100644 --- a/tests/core/controllers/test_trainer_paddle.py +++ b/tests/core/controllers/test_trainer_paddle.py @@ -1,3 +1,5 @@ +import os +from typing import List import pytest from dataclasses import dataclass @@ -5,6 +7,7 @@ from fastNLP.core.controllers.trainer import Trainer from fastNLP.core.metrics.accuracy import Accuracy from fastNLP.core.callbacks.progress_callback import RichCallback from fastNLP.envs.imports import _NEED_IMPORT_PADDLE +from fastNLP.envs.env import USER_CUDA_VISIBLE_DEVICES if _NEED_IMPORT_PADDLE: from paddle.optimizer import Adam @@ -34,6 +37,8 @@ def test_trainer_paddle( callbacks, n_epochs=2, ): + if isinstance(device, List) and USER_CUDA_VISIBLE_DEVICES not in os.environ: + pytest.skip("Skip test fleet if FASTNLP_BACKEND is not set to paddle.") model = PaddleNormalModel_Classification_1( num_labels=TrainPaddleConfig.num_labels, feature_dimension=TrainPaddleConfig.feature_dimension diff --git a/tests/core/utils/test_paddle_utils.py b/tests/core/utils/test_paddle_utils.py index d86d215f..96a3b41a 100644 --- a/tests/core/utils/test_paddle_utils.py +++ b/tests/core/utils/test_paddle_utils.py @@ -6,33 +6,38 @@ from fastNLP.core.utils.paddle_utils import get_device_from_visible, paddle_to, from fastNLP.envs.imports import _NEED_IMPORT_PADDLE if _NEED_IMPORT_PADDLE: import paddle + @pytest.mark.parametrize( - ("user_visible_devices, cuda_visible_devices, device, output_type, correct"), + ("user_visible_devices, cuda_visible_devices, device, correct"), ( - ("0,1,2,3,4,5,6,7", "0", "cpu", str, "cpu"), - ("0,1,2,3,4,5,6,7", "0", "cpu", int, "cpu"), - ("0,1,2,3,4,5,6,7", "3,4,5", "gpu:4", int, 1), - ("0,1,2,3,4,5,6,7", "3,4,5", "gpu:5", str, "gpu:2"), - ("3,4,5,6", "3,5", 0, int, 0), - ("3,6,7,8", "6,7,8", "gpu:2", str, "gpu:1"), + (None, None, 1, "gpu:1"), + (None, "2,4,5,6", 5, "gpu:2"), + (None, "3,4,5", 4, "gpu:1"), + ("0,1,2,3,4,5,6,7", "0", "cpu", "cpu"), + ("0,1,2,3,4,5,6,7", "0", "cpu", "cpu"), + ("0,1,2,3,4,5,6,7", "3,4,5", "gpu:4", "gpu:1"), + ("0,1,2,3,4,5,6,7", "3,4,5", "gpu:5", "gpu:2"), + ("3,4,5,6", "3,5", 0, "gpu:0"), + ("3,6,7,8", "6,7,8", "gpu:2", "gpu:1"), ) ) -@pytest.mark.paddle -def test_get_device_from_visible(user_visible_devices, cuda_visible_devices, device, output_type, correct): +def test_get_device_from_visible(user_visible_devices, cuda_visible_devices, device, correct): _cuda_visible_devices = os.getenv("CUDA_VISIBLE_DEVICES") _user_visible_devices = os.getenv("USER_CUDA_VISIBLE_DEVICES") - os.environ["CUDA_VISIBLE_DEVICES"] = cuda_visible_devices - os.environ["USER_CUDA_VISIBLE_DEVICES"] = user_visible_devices - res = get_device_from_visible(device, output_type) + if cuda_visible_devices is not None: + os.environ["CUDA_VISIBLE_DEVICES"] = cuda_visible_devices + if user_visible_devices is not None: + os.environ["USER_CUDA_VISIBLE_DEVICES"] = user_visible_devices + res = get_device_from_visible(device) assert res == correct # 还原环境变量 if _cuda_visible_devices is None: - del os.environ["CUDA_VISIBLE_DEVICES"] + os.environ.pop("CUDA_VISIBLE_DEVICES", None) else: os.environ["CUDA_VISIBLE_DEVICES"] = _cuda_visible_devices if _user_visible_devices is None: - del os.environ["USER_CUDA_VISIBLE_DEVICES"] + os.environ.pop("USER_CUDA_VISIBLE_DEVICES", None) else: os.environ["USER_CUDA_VISIBLE_DEVICES"] = _user_visible_devices From a34a40dfae3b43462b87e9dd475c3ae5d5e200ca Mon Sep 17 00:00:00 2001 From: x54-729 <17307130121@fudan.edu.cn> Date: Mon, 9 May 2022 19:56:15 +0000 Subject: [PATCH 08/15] =?UTF-8?q?=E5=9C=A8=E5=88=86=E5=B8=83=E5=BC=8F?= =?UTF-8?q?=E8=AE=AD=E7=BB=83=E4=B8=AD=EF=BC=8C=E4=B8=BAUSER=5FCUDA=5FVISI?= =?UTF-8?q?BLE=5FDEVICES=E4=B8=BANone=E7=9A=84=E6=83=85=E5=86=B5=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=E6=8F=90=E9=86=92?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/core/drivers/paddle_driver/fleet.py | 4 ++-- .../core/drivers/paddle_driver/initialize_paddle_driver.py | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/fastNLP/core/drivers/paddle_driver/fleet.py b/fastNLP/core/drivers/paddle_driver/fleet.py index d09cacc1..59c1e0ae 100644 --- a/fastNLP/core/drivers/paddle_driver/fleet.py +++ b/fastNLP/core/drivers/paddle_driver/fleet.py @@ -191,8 +191,8 @@ class PaddleFleetDriver(PaddleDriver): 不管是什么情况，`PaddleFleetDriver` 在 `setup` 函数的最后，都会将所有进程的 pid 主动记录下来，这样当一个进程出现 exception 后， driver 的 on_exception 函数就会被 trainer 调用，其会调用 os.kill 指令将其它进程 kill 掉； """ - # if USER_CUDA_VISIBLE_DEVICES not in os.environ: - # raise RuntimeError("To run paddle distributed training, please set `FASTNLP_BACKEND` to 'paddle' before using FastNLP.") + if USER_CUDA_VISIBLE_DEVICES not in os.environ: + raise RuntimeError("To run paddle distributed training, please set `FASTNLP_BACKEND` to 'paddle' before using FastNLP.") super(PaddleFleetDriver, self).__init__(model, fp16=fp16, **kwargs) # 如果不是通过 launch 启动，要求用户必须传入 parallel_device diff --git a/fastNLP/core/drivers/paddle_driver/initialize_paddle_driver.py b/fastNLP/core/drivers/paddle_driver/initialize_paddle_driver.py index aa1b2db5..54ede2d8 100644 --- a/fastNLP/core/drivers/paddle_driver/initialize_paddle_driver.py +++ b/fastNLP/core/drivers/paddle_driver/initialize_paddle_driver.py @@ -7,6 +7,7 @@ from .single_device import PaddleSingleDriver from .fleet import PaddleFleetDriver from fastNLP.envs.imports import _NEED_IMPORT_PADDLE +from fastNLP.envs.env import USER_CUDA_VISIBLE_DEVICES from fastNLP.core.utils import is_in_paddle_launch_dist, get_paddle_gpu_str from fastNLP.core.log import logger @@ -30,8 +31,10 @@ def initialize_paddle_driver(driver: str, device: Optional[Union[str, int, List[ """ if driver != "paddle": raise ValueError("When initialize PaddleDriver, parameter `driver` must be 'paddle'.") - user_visible_devices = os.getenv("USER_CUDA_VISIBLE_DEVICES") + user_visible_devices = os.getenv(USER_CUDA_VISIBLE_DEVICES) if is_in_paddle_launch_dist(): + if user_visible_devices is None: + raise RuntimeError("To run paddle distributed training, please set `FASTNLP_BACKEND` to 'paddle' before using FastNLP.") if device is not None: logger.warning_once("Parameter `device` would be ignored when you are using `paddle.distributed.launch` to pull " "up your script. And we will directly get the local device via environment variables.") @@ -65,6 +68,7 @@ def initialize_paddle_driver(driver: str, device: Optional[Union[str, int, List[ device = [get_paddle_gpu_str(g) for g in device] elif device is not None and not isinstance(device, str): raise ValueError("Parameter `device` is wrong type, please check our documentation for the right use.") + if isinstance(device, List): return PaddleFleetDriver(model, device, **kwargs) else: From 7763b2e087188cabbf0bb68270553f79d98fc187 Mon Sep 17 00:00:00 2001 From: yh_cc Date: Tue, 10 May 2022 11:51:12 +0800 Subject: [PATCH 09/15] =?UTF-8?q?=E6=96=B0=E5=A2=9ECallback=20on=5Fload=5F?= =?UTF-8?q?checkpoint=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/core/__init__.py | 3 - fastNLP/core/callbacks/callback_manager.py | 10 +-- .../test_checkpoint_callback_torch.py | 73 +++++++++++++++++++ 3 files changed, 77 insertions(+), 9 deletions(-) diff --git a/fastNLP/core/__init__.py b/fastNLP/core/__init__.py index b0f71f52..8800e1d6 100644 --- a/fastNLP/core/__init__.py +++ b/fastNLP/core/__init__.py @@ -3,9 +3,7 @@ __all__ = [ 'Callback', 'Event', 'Filter', - 'CallbackManager', 'CheckpointCallback', - 'choose_progress_callback', 'ProgressCallback', 'RichCallback', "LRSchedCallback", @@ -54,7 +52,6 @@ __all__ = [ 'DataSet', 'FieldArray', 'Instance', - 'ApplyResultException', # drivers "TorchSingleDriver", diff --git a/fastNLP/core/callbacks/callback_manager.py b/fastNLP/core/callbacks/callback_manager.py index 82b1a756..27770115 100644 --- a/fastNLP/core/callbacks/callback_manager.py +++ b/fastNLP/core/callbacks/callback_manager.py @@ -180,8 +180,8 @@ class CallbackManager: states[each_callback.callback_name]["states"] = each_callback.on_save_checkpoint(trainer) if len(_duplicated_callbacks) > 0: - logger.warning(f"Notice these callbacks' `callback_name` are duplicated: {_duplicated_callbacks}, " - f"and we will only save the first callback's state we meet.") + logger.warning(f"Notice these callback_name: {_duplicated_callbacks} are duplicated, " + f"fastNLP will only save the first callback's state.") # 2. 每一个具体的 callback 函数的 filter 的状态； _record_duplicated_callback_names = set() @@ -223,8 +223,8 @@ class CallbackManager: _duplicated_callback_names.add(each_callback_filters[0]) if len(_duplicated_callback_names) > 0: - logger.warning(f"Notice these callbacks' `callback_name` are duplicated: {_duplicated_callback_names}, " - f"and we will only load the first callback's state we meet.") + logger.rank_zero_warning(f"Notice these callback_name: {_duplicated_callback_names} are duplicated, " + f"fastNLP will only load the first callback's state.") # 2. 再恢复每一个 callback 的单独的状态； # 每一个我们自己提供的类 callback，都需要重写其特定的 `callback_name` 方法，保证如果两个 callback 的 callback_name 一样， @@ -235,8 +235,6 @@ class CallbackManager: _already_loaded_callback_names.add(each_callback.callback_name) # 这里要注意，我们已经确保每一个 callback 的 `on_load_checkpoint` 函数拿到的就是其自己的状态； each_callback.on_load_checkpoint(trainer, states[each_callback.callback_name]["states"]) - else: - each_callback.on_load_checkpoint(trainer, None) @property def has_trainer_checkpoint(self) -> bool: diff --git a/tests/core/callbacks/test_checkpoint_callback_torch.py b/tests/core/callbacks/test_checkpoint_callback_torch.py index 60dcc862..3105acba 100644 --- a/tests/core/callbacks/test_checkpoint_callback_torch.py +++ b/tests/core/callbacks/test_checkpoint_callback_torch.py @@ -14,6 +14,7 @@ from tests.helpers.utils import magic_argv_env_context from fastNLP.envs.distributed import rank_zero_rm from tests.helpers.models.torch_model import TorchNormalModel_Classification_1 from tests.helpers.datasets.torch_data import TorchArgMaxDataset +from tests.helpers.utils import Capturing from torchmetrics import Accuracy from fastNLP.core.log import logger @@ -428,6 +429,78 @@ def test_trainer_checkpoint_callback_1( dist.destroy_process_group() +@pytest.mark.torch +def test_load_state(model_and_optimizers): + try: + path = Path.cwd().joinpath(f"test_model_checkpoint") + path.mkdir(exist_ok=True, parents=True) + from fastNLP import Event, Callback + @Trainer.on(Event.on_before_backward(every=3), marker='all') + def print_outputs(*args): + print("????") + + class StateCallback(Callback): + def __init__(self, name): + self.name = name + + def on_save_checkpoint(self, trainer): + return {'name': self.name} + + def on_load_checkpoint(self, trainer, states): + self.name = states['name'] + + def on_train_end(self, trainer): + print(self.name) + + callbacks = [StateCallback('old_callback1'), StateCallback('old_callback2'), + CheckpointCallback(folder=path, every_n_epochs=1, save_object='trainer')] + + trainer = Trainer( + model=model_and_optimizers.model, + driver='torch', + device='cpu', + optimizers=model_and_optimizers.optimizers, + train_dataloader=model_and_optimizers.train_dataloader, + evaluate_dataloaders=model_and_optimizers.evaluate_dataloaders, + input_mapping=model_and_optimizers.input_mapping, + output_mapping=model_and_optimizers.output_mapping, + metrics=model_and_optimizers.metrics, + n_epochs=3, + callbacks=callbacks, + output_from_new_proc="all" + ) + trainer.run(num_eval_sanity_batch=0, num_train_batch_per_epoch=2) + + all_saved_model_paths = {w.name: w for w in path.joinpath(os.environ[FASTNLP_LAUNCH_TIME]).iterdir()} + epoch_2_path = all_saved_model_paths['trainer-epoch_2'] + + callbacks = [StateCallback('new_callback1'), StateCallback('new_callback2')] + trainer = Trainer( + model=model_and_optimizers.model, + driver='torch', + device='cpu', + optimizers=model_and_optimizers.optimizers, + train_dataloader=model_and_optimizers.train_dataloader, + evaluate_dataloaders=model_and_optimizers.evaluate_dataloaders, + input_mapping=model_and_optimizers.input_mapping, + output_mapping=model_and_optimizers.output_mapping, + metrics=model_and_optimizers.metrics, + n_epochs=3, + callbacks=callbacks, + output_from_new_proc="all" + ) + trainer.load(folder=epoch_2_path) + with Capturing() as output: + trainer.run(num_eval_sanity_batch=0, num_train_batch_per_epoch=2) + + assert 'old_callback1' in output[0] + assert 'new_callback2' in output[0] + assert output[0].count('???')==1 + + finally: + rank_zero_rm(path) + + @pytest.mark.torch # 通过自己编写 model_save_fn 和 model_load_fn 来测试 huggingface 的 transformers 的模型的保存和加载； @pytest.mark.parametrize("driver,device", [("torch_ddp", [6, 7]), ("torch", 7)]) # ("torch", "cpu"), ("torch_ddp", [0, 1]), ("torch", 1) From 4885b8237c02f6a15ecaf39de3f1f93d0d00fc31 Mon Sep 17 00:00:00 2001 From: x54-729 <17307130121@fudan.edu.cn> Date: Tue, 10 May 2022 07:53:39 +0000 Subject: [PATCH 10/15] =?UTF-8?q?=E5=9C=A8=20Trainer=20=E4=B8=AD=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=E5=AF=B9=20paddle=5Fkwargs=20=E7=9A=84=E8=AF=B4?= =?UTF-8?q?=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/core/controllers/trainer.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/fastNLP/core/controllers/trainer.py b/fastNLP/core/controllers/trainer.py index 2116674f..afba4de8 100644 --- a/fastNLP/core/controllers/trainer.py +++ b/fastNLP/core/controllers/trainer.py @@ -282,32 +282,42 @@ class Trainer(TrainerEventTrigger): :kwargs: * *torch_kwargs* -- 用于在指定 ``driver`` 为 'torch' 时设定具体 driver 实例的一些参数： + * ddp_kwargs -- 用于在使用 ``TorchDDPDriver`` 时指定 ``DistributedDataParallel`` 初始化时的参数；例如传入 - {'find_unused_parameters': True} 来解决有参数不参与前向运算导致的报错等； + {'find_unused_parameters': True} 来解决有参数不参与前向运算导致的报错等； * set_grad_to_none -- 是否在训练过程中在每一次 optimizer 更新后将 grad 置为 None； * torch_non_blocking -- 表示用于 pytorch 的 tensor 的 to 方法的参数 non_blocking； + * *paddle_kwargs* -- 用于在指定 ``driver`` 为 'paddle' 时设定具体 driver 实例的一些参数： + + * fleet_kwargs -- 用于在使用 ``PaddleFleetDriver`` 时指定 ``DataParallel`` 和 ``fleet`` 初始化时的参数，包括： + + * is_collective -- 是否使用 paddle 集群式的分布式训练方法，目前仅支持为 True 的情况； + * role_maker -- 初始化 ``fleet`` 分布式训练 API 时使用的 ``RoleMaker`` + * 其它用于初始化 ``DataParallel`` 的参数； * *data_device* -- 一个具体的 driver 实例中，有 ``model_device`` 和 ``data_device``，前者表示模型所在的设备，后者表示 - 当 ``model_device`` 为 None 时应当将数据迁移到哪个设备； + 当 ``model_device`` 为 None 时应当将数据迁移到哪个设备； - .. note:: + .. note:: 注意您在绝大部分情况下不会用到该参数！ 1. 当 driver 实例的 ``model_device`` 不为 None 时，该参数无效； 2. 对于 pytorch，仅当用户自己通过 ``python -m torch.distributed.launch`` 并且自己初始化 ``init_process_group`` 时， driver 实例的 ``model_device`` 才会为 None； + 3. 对于 paddle，仅当用户自己通过 ``python -m paddle.distributed.launch`` 并且自己初始化 :func:`~init_parallel_env` 或 + :meth:`fleet.init` 时，driver 实例的 ``model_device`` 才会为 None； * *use_dist_sampler* -- 表示是否使用分布式的 ``sampler``。在多卡时，分布式 ``sampler`` 将自动决定每张卡上读取的 sample ，使得一个 epoch - 内所有卡的 sample 加起来为一整个数据集的 sample。默认会根据 driver 是否为分布式进行设置。 + 内所有卡的 sample 加起来为一整个数据集的 sample。默认会根据 driver 是否为分布式进行设置。 * *evaluate_use_dist_sampler* -- 表示在 ``Evaluator`` 中在使用分布式的时候是否将 dataloader 的 ``sampler`` 替换为分布式的 ``sampler``；默认为 ``True``； * *output_from_new_proc* -- 应当为一个字符串，表示在多进程的 driver 中其它进程的输出流应当被做如何处理；其值应当为以下之一： - ["all", "ignore", "only_error"]；当该参数的值不是以上值时，该值应当表示一个文件夹的名字，我们会将其他 rank 的输出流重定向到 - log 文件中，然后将 log 文件保存在通过该参数值设定的文件夹中；默认为 "only_error"； + ["all", "ignore", "only_error"]；当该参数的值不是以上值时，该值应当表示一个文件夹的名字，我们会将其他 rank 的输出流重定向到 + log 文件中，然后将 log 文件保存在通过该参数值设定的文件夹中；默认为 "only_error"； - 注意该参数仅当使用分布式的 ``driver`` 时才有效，例如 ``TorchDDPDriver``； + 注意该参数仅当使用分布式的 ``driver`` 时才有效，例如 ``TorchDDPDriver``； * *progress_bar* -- 以哪种方式显示 progress ，目前支持[None, 'raw', 'rich', 'auto'] 或者 RichCallback, RawTextCallback对象， - 默认为 auto , auto 表示如果检测到当前 terminal 为交互型则使用 RichCallback，否则使用 RawTextCallback对象。如果 - 需要定制 progress bar 的参数，例如打印频率等，可以传入 RichCallback, RawTextCallback 对象。 + 默认为 auto , auto 表示如果检测到当前 terminal 为交互型则使用 RichCallback，否则使用 RawTextCallback对象。如果 + 需要定制 progress bar 的参数，例如打印频率等，可以传入 RichCallback, RawTextCallback 对象。 * *train_input_mapping* -- 与 input_mapping 一致，但是只用于 ``Trainer`` 中。与 input_mapping 互斥。 * *train_output_mapping* -- 与 output_mapping 一致，但是只用于 ``Trainer`` 中。与 output_mapping 互斥。 * *evaluate_input_mapping* -- 与 input_mapping 一致，但是只用于 ``Evaluator`` 中。与 input_mapping 互斥。 From 4fbca222671d18759c5bfe716e045ef7a4a4445e Mon Sep 17 00:00:00 2001 From: x54-729 <17307130121@fudan.edu.cn> Date: Tue, 10 May 2022 07:54:17 +0000 Subject: [PATCH 11/15] small --- fastNLP/core/drivers/choose_driver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fastNLP/core/drivers/choose_driver.py b/fastNLP/core/drivers/choose_driver.py index 294bfe28..4be1e502 100644 --- a/fastNLP/core/drivers/choose_driver.py +++ b/fastNLP/core/drivers/choose_driver.py @@ -23,9 +23,9 @@ def choose_driver(model, driver: Union[str, Driver], device: Optional[Union[int, elif driver in {"jittor"}: from fastNLP.core.drivers.jittor_driver.initialize_jittor_driver import initialize_jittor_driver return initialize_jittor_driver(driver, device, model, **kwargs) - elif driver in {"paddle", "fleet"}: + elif driver in {"paddle"}: from fastNLP.core.drivers.paddle_driver.initialize_paddle_driver import initialize_paddle_driver return initialize_paddle_driver(driver, device, model, **kwargs) else: raise ValueError("Parameter `driver` can only be one of these values: ['torch', 'fairscale', " - "'jittor', 'paddle', 'fleet'].") \ No newline at end of file + "'jittor', 'paddle'].") \ No newline at end of file From 9a15af88d7f397c4ddd280db71b9dd3e93f6abcf Mon Sep 17 00:00:00 2001 From: x54-729 <17307130121@fudan.edu.cn> Date: Tue, 10 May 2022 08:10:48 +0000 Subject: [PATCH 12/15] =?UTF-8?q?=E5=A6=82=E6=9E=9C=E7=94=A8=E6=88=B7?= =?UTF-8?q?=E6=B2=A1=E6=9C=89=E8=AE=BE=E7=BD=AEbackend=E4=B8=94=E7=94=A8la?= =?UTF-8?q?unch=E5=90=AF=E5=8A=A8=E4=BA=86=E5=A4=9A=E5=8D=A1=EF=BC=8C?= =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E7=9B=B8=E5=BA=94=E7=9A=84=E6=8F=90=E9=86=92?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/envs/set_backend.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fastNLP/envs/set_backend.py b/fastNLP/envs/set_backend.py index e6b9bf59..d925d282 100644 --- a/fastNLP/envs/set_backend.py +++ b/fastNLP/envs/set_backend.py @@ -101,6 +101,11 @@ def _set_backend(): elif backend == 'torch': assert _module_available(backend), f"You must have {backend} available to use {backend} backend." + if 'PADDLE_RANK_IN_NODE' in os.environ and 'FLAGS_selected_gpus' in os.environ \ + and "USER_CUDA_VISIBLE_DEVICES" not in os.environ: + # 当用户没有设置 backend 并且使用 launch 启动了多卡，应该提醒用户进行设置 + raise RuntimeError("To run paddle distributed training, please set `FASTNLP_BACKEND` to 'paddle' before using FastNLP.") + def set_env(global_seed=None): """ From d79de6b008c2395eb1743a0106fae58e00b87b9f Mon Sep 17 00:00:00 2001 From: x54-729 <17307130121@fudan.edu.cn> Date: Tue, 10 May 2022 10:58:40 +0000 Subject: [PATCH 13/15] =?UTF-8?q?1.=20=E7=BB=A7=E7=BB=AD=E5=AE=8C=E5=96=84?= =?UTF-8?q?=E9=83=A8=E5=88=86=E6=96=87=E6=A1=A3=EF=BC=9B2.=E5=88=A0?= =?UTF-8?q?=E9=99=A4=20paddle=20=E5=A4=9A=E5=8D=A1=E4=B8=8B=E7=9A=84=20dat?= =?UTF-8?q?a=5Fdevice=20=E5=8A=9F=E8=83=BD=203.=20=E5=B0=86=20paddle=5Futi?= =?UTF-8?q?ls=20=E4=B8=8B=E7=9A=84=20get=5Fdevice=5Ffrom=5Fvisible=20?= =?UTF-8?q?=E5=87=BD=E6=95=B0=E6=9B=B4=E5=90=8D=E4=B8=BA=20=5Fconvert=5Fda?= =?UTF-8?q?ta=5Fdevice=20=E5=B9=B6=E8=BF=9B=E8=A1=8C=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/core/controllers/trainer.py | 3 +- .../core/dataloaders/paddle_dataloader/fdl.py | 2 +- .../core/dataloaders/torch_dataloader/fdl.py | 2 +- fastNLP/core/dataset/dataset.py | 26 ++-- .../jittor_driver/initialize_jittor_driver.py | 16 ++- .../drivers/jittor_driver/jittor_driver.py | 12 +- fastNLP/core/drivers/jittor_driver/mpi.py | 8 ++ .../drivers/jittor_driver/single_device.py | 18 ++- fastNLP/core/drivers/jittor_driver/utils.py | 3 +- fastNLP/core/drivers/paddle_driver/fleet.py | 26 +--- .../paddle_driver/initialize_paddle_driver.py | 20 +-- .../drivers/paddle_driver/paddle_driver.py | 8 +- .../drivers/paddle_driver/single_device.py | 7 +- .../torch_driver/initialize_torch_driver.py | 11 +- .../metrics/backend/paddle_backend/backend.py | 5 +- fastNLP/core/utils/__init__.py | 3 +- fastNLP/core/utils/jittor_utils.py | 9 +- fastNLP/core/utils/paddle_utils.py | 122 ++++++++++-------- fastNLP/core/utils/torch_utils.py | 10 +- fastNLP/core/utils/utils.py | 86 ++++++------ .../_test_trainer_fleet_outside.py | 2 - tests/core/utils/test_paddle_utils.py | 12 +- 22 files changed, 223 insertions(+), 188 deletions(-) diff --git a/fastNLP/core/controllers/trainer.py b/fastNLP/core/controllers/trainer.py index afba4de8..0609ac12 100644 --- a/fastNLP/core/controllers/trainer.py +++ b/fastNLP/core/controllers/trainer.py @@ -304,8 +304,7 @@ class Trainer(TrainerEventTrigger): 1. 当 driver 实例的 ``model_device`` 不为 None 时，该参数无效； 2. 对于 pytorch，仅当用户自己通过 ``python -m torch.distributed.launch`` 并且自己初始化 ``init_process_group`` 时， driver 实例的 ``model_device`` 才会为 None； - 3. 对于 paddle，仅当用户自己通过 ``python -m paddle.distributed.launch`` 并且自己初始化 :func:`~init_parallel_env` 或 - :meth:`fleet.init` 时，driver 实例的 ``model_device`` 才会为 None； + 3. 对于 paddle，该参数无效； * *use_dist_sampler* -- 表示是否使用分布式的 ``sampler``。在多卡时，分布式 ``sampler`` 将自动决定每张卡上读取的 sample ，使得一个 epoch 内所有卡的 sample 加起来为一整个数据集的 sample。默认会根据 driver 是否为分布式进行设置。 diff --git a/fastNLP/core/dataloaders/paddle_dataloader/fdl.py b/fastNLP/core/dataloaders/paddle_dataloader/fdl.py index 342a6c19..393324d4 100644 --- a/fastNLP/core/dataloaders/paddle_dataloader/fdl.py +++ b/fastNLP/core/dataloaders/paddle_dataloader/fdl.py @@ -164,7 +164,7 @@ class PaddleDataLoader(DataLoader): """ 获取当前 ``batch`` 中每条数据对应的索引。 - :return: 当前 ``batch`` 数据的索引 + :return: 当前 ``batch`` 数据的索引； """ return self.cur_batch_indices diff --git a/fastNLP/core/dataloaders/torch_dataloader/fdl.py b/fastNLP/core/dataloaders/torch_dataloader/fdl.py index 1f737467..456af44f 100644 --- a/fastNLP/core/dataloaders/torch_dataloader/fdl.py +++ b/fastNLP/core/dataloaders/torch_dataloader/fdl.py @@ -172,7 +172,7 @@ class TorchDataLoader(DataLoader): """ 获取当前 ``batch`` 中每条数据对应的索引。 - :return: 当前 ``batch`` 数据的索引 + :return: 当前 ``batch`` 数据的索引； """ return self.cur_batch_indices diff --git a/fastNLP/core/dataset/dataset.py b/fastNLP/core/dataset/dataset.py index 6b908c6a..c592984f 100644 --- a/fastNLP/core/dataset/dataset.py +++ b/fastNLP/core/dataset/dataset.py @@ -400,16 +400,22 @@ class DataSet: new_field_name: str = None, num_proc: int = 0, progress_desc: str = None, show_progress_bar: bool = True): r""" - 将 :class:`~DataSet` 每个 ``instance`` 中为 ``field_name`` 的 ``field`` 传给函数 ``func``，并获取函数的返回值。 - - :param field_name: 传入 ``func`` 的 ``field`` 名称。 - :param func: 一个函数，其输入是 ``instance`` 中名为 ``field_name`` 的 ``field`` 的内容。 - :param new_field_name: 将 ``func`` 返回的内容放入到 ``new_field_name`` 对应的 ``field`` 中，如果名称与已有的 ``field`` 相同 - 则进行覆盖。如果为 ``None`` 则不会覆盖和创建 ``field`` 。 - :param num_proc: 使用进程的数量。请注意，由于 ``python`` 语言的特性，使用了多少进程就会导致多少倍内存的增长。 - :param progress_desc: 进度条的描述字符，默认为 ``Main``。 - :param show_progress_bar: 是否展示进度条；默认为展示。 - :return: 从函数 ``func`` 中得到的返回值。 + 将 :class:`~DataSet` 每个 ``instance`` 中为 ``field_name`` 的 ``field`` 传给函数 ``func``，并写入到 ``new_field_name`` + 中。 + + :param field_name: 传入 ``func`` 的 ``field`` 名称； + :param func: 对指定 ``field`` 进行处理的函数，注意其输入应为 ``instance`` 中名为 ``field_name`` 的 ``field`` 的内容； + :param new_field_name: 函数执行结果写入的 ``field`` 名称。该函数会将 ``func`` 返回的内容放入到 ``new_field_name`` 对 + 应的 ``field`` 中，注意如果名称与已有的 ``field`` 相同则会进行覆盖。如果为 ``None`` 则不会覆盖和创建 ``field`` ； + :param num_proc: 使用进程的数量。 + + .. note:: + + 由于 ``python`` 语言的特性，设置该参数后会导致相应倍数的内存增长，这可能会对您程序的执行带来一定的影响。 + + :param progress_desc: 进度条的描述字符，默认为 ``Main``； + :param show_progress_bar: 是否在处理过程中展示进度条； + :return: 从函数 ``func`` 中得到的返回值； """ assert len(self) != 0, "Null DataSet cannot use apply_field()." if not self.has_field(field_name=field_name): diff --git a/fastNLP/core/drivers/jittor_driver/initialize_jittor_driver.py b/fastNLP/core/drivers/jittor_driver/initialize_jittor_driver.py index e2d8aadb..4b1fcba7 100644 --- a/fastNLP/core/drivers/jittor_driver/initialize_jittor_driver.py +++ b/fastNLP/core/drivers/jittor_driver/initialize_jittor_driver.py @@ -7,18 +7,22 @@ from fastNLP.envs.imports import _NEED_IMPORT_JITTOR if _NEED_IMPORT_JITTOR: import jittor +__all__ = [] + def initialize_jittor_driver(driver: str, device: Union[str, int, List[int]], model: jittor.Module, **kwargs) -> JittorDriver: r""" - 用来根据参数 `driver` 和 `device` 来确定并且初始化一个具体的 `Driver` 实例然后返回回去； - 在这个函数中，我们会根据用户设置的device来确定JittorDriver的mode。 + 用来根据参数 ``device`` 来确定并且初始化一个具体的 ``Driver`` 实例然后返回回去。 + + .. todo:: + + 创建多卡的 driver - :param driver: 该参数的值应为以下之一：["jittor"]； - :param device: jittor运行的设备 + :param driver: 该参数的值应为以下之一：``["jittor"]``； + :param device: ``jittor`` 运行的设备； :param model: 训练或者评测的具体的模型； :param kwargs: - :return: 返回一个元组，元组的第一个值是具体的基于 jittor 的 `Driver` 实例，元组的第二个值是该 driver 的名字（用于检测一个脚本中 - 先后 driver 的次序的正确问题）； + :return: :class:`~fastNLP.core.JittorSingleDriver` 或 :class:`~fastNLP.core.JittorMPIDriver` 实例； """ if driver not in {"jittor"}: diff --git a/fastNLP/core/drivers/jittor_driver/jittor_driver.py b/fastNLP/core/drivers/jittor_driver/jittor_driver.py index b751354d..7efff348 100644 --- a/fastNLP/core/drivers/jittor_driver/jittor_driver.py +++ b/fastNLP/core/drivers/jittor_driver/jittor_driver.py @@ -24,7 +24,17 @@ if _NEED_IMPORT_JITTOR: class JittorDriver(Driver): r""" - Jittor 框架的 Driver + ``Jittor`` 框架的 ``Driver`` + + .. note:: + + 这是一个正在开发中的功能，敬请期待。 + + .. todo:: + + 实现 fp16 的设置，且支持 cpu 和 gpu 的切换； + 实现用于断点重训的 save 和 load 函数； + """ def __init__(self, model, fp16: bool = False, **kwargs): diff --git a/fastNLP/core/drivers/jittor_driver/mpi.py b/fastNLP/core/drivers/jittor_driver/mpi.py index bb52f67d..bfa49e68 100644 --- a/fastNLP/core/drivers/jittor_driver/mpi.py +++ b/fastNLP/core/drivers/jittor_driver/mpi.py @@ -13,6 +13,14 @@ __all__ = [ ] class JittorMPIDriver(JittorDriver): + """ + 执行 ``Jittor`` 框架下分布式训练的 ``Driver``。 + + .. note:: + + 这是一个正在开发中的功能，敬请期待。 + + """ def __init__( self, model, diff --git a/fastNLP/core/drivers/jittor_driver/single_device.py b/fastNLP/core/drivers/jittor_driver/single_device.py index ab1e8595..be704e69 100644 --- a/fastNLP/core/drivers/jittor_driver/single_device.py +++ b/fastNLP/core/drivers/jittor_driver/single_device.py @@ -16,8 +16,17 @@ __all__ = [ class JittorSingleDriver(JittorDriver): r""" - 用于 cpu 和 单卡 gpu 运算 - TODO: jittor 的 fp16 + ``Jittor`` 框架下用于 ``cpu`` 和单卡 ``gpu`` 运算的 ``Driver``。 + + .. note:: + + 这是一个正在开发中的功能，敬请期待。 + + .. todo:: + + 支持 cpu 和 gpu 的切换； + 实现断点重训中替换 dataloader 的 set_dist_repro_dataloader 函数 + """ def __init__(self, model, device=None, fp16: bool = False, **kwargs): @@ -30,11 +39,6 @@ class JittorSingleDriver(JittorDriver): self.world_size = 1 def step(self): - """ - jittor optimizers 的step函数可以传入参数loss - 此时会同时进行 zero_grad 和 backward - 为了统一，这里暂不使用这样的方式 - """ for optimizer in self.optimizers: optimizer.step() diff --git a/fastNLP/core/drivers/jittor_driver/utils.py b/fastNLP/core/drivers/jittor_driver/utils.py index f8ddbbe1..43be9ac3 100644 --- a/fastNLP/core/drivers/jittor_driver/utils.py +++ b/fastNLP/core/drivers/jittor_driver/utils.py @@ -5,10 +5,11 @@ from fastNLP.envs.imports import _NEED_IMPORT_JITTOR if _NEED_IMPORT_JITTOR: import jittor +__all__ = [] + class DummyGradScaler: """ 用于仿造的GradScaler对象，防止重复写大量的if判断 - """ def __init__(self, *args, **kwargs): pass diff --git a/fastNLP/core/drivers/paddle_driver/fleet.py b/fastNLP/core/drivers/paddle_driver/fleet.py index 59c1e0ae..03dc3375 100644 --- a/fastNLP/core/drivers/paddle_driver/fleet.py +++ b/fastNLP/core/drivers/paddle_driver/fleet.py @@ -1,8 +1,6 @@ import os from typing import List, Union, Optional, Dict, Tuple, Callable -from fastNLP.core.utils.paddle_utils import get_device_from_visible - from .paddle_driver import PaddleDriver from .fleet_launcher import FleetLauncher from .utils import ( @@ -21,6 +19,7 @@ from fastNLP.core.utils import ( is_in_paddle_dist, get_paddle_device_id, ) +from fastNLP.core.utils.paddle_utils import _convert_data_device from fastNLP.envs.distributed import rank_zero_rm from fastNLP.core.samplers import ( ReproduceBatchSampler, @@ -221,25 +220,6 @@ class PaddleFleetDriver(PaddleDriver): "you initialize the paddle distribued process out of our control.") self.outside_fleet = True - # 用户只有将模型上传到对应机器上后才能用 DataParallel 包裹，因此如果用户在外面初始化了 Fleet，那么在 PaddleFleetDriver 中 - # 我们就直接将 model_device 置为 None； - self._model_device = None - - # 当参数 `device` 为 None 时并且该参数不为 None，表示将对应的数据移到指定的机器上； - self._data_device = kwargs.get("data_device", None) - if self._data_device is not None: - if isinstance(self._data_device, int): - if self._data_device < 0: - raise ValueError("Parameter `data_device` can not be smaller than 0.") - _could_use_device_num = paddle.device.cuda.device_count() - if self._data_device >= _could_use_device_num: - raise ValueError("The gpu device that parameter `device` specifies is not existed.") - self._data_device = f"gpu:{self._data_device}" - elif not isinstance(self._data_device, str): - raise ValueError("Parameter `device` is wrong type, please check our documentation for the right use.") - # if self.outside_fleet and paddle.device.get_device() != self._data_device: - # logger.warning("`Parameter data_device` is not equal to paddle.deivce.get_device(), " - # "please keep them equal to avoid some potential bugs.") self.world_size = None self.global_rank = 0 @@ -419,8 +399,6 @@ class PaddleFleetDriver(PaddleDriver): @property def data_device(self): - if self.outside_fleet: - return self._data_device return self.model_device def model_call(self, batch, fn: Callable, signature_fn: Optional[Callable]) -> Dict: @@ -574,7 +552,7 @@ class PaddleFleetDriver(PaddleDriver): def broadcast_object(self, obj, src:int=0, group=None, **kwargs): # 因为设置了CUDA_VISIBLE_DEVICES，可能会引起错误 - device = get_device_from_visible(self.data_device) + device = _convert_data_device(self.data_device) return fastnlp_paddle_broadcast_object(obj, src, device=device, group=group) def all_gather(self, obj, group=None) -> List: diff --git a/fastNLP/core/drivers/paddle_driver/initialize_paddle_driver.py b/fastNLP/core/drivers/paddle_driver/initialize_paddle_driver.py index 54ede2d8..22098ff2 100644 --- a/fastNLP/core/drivers/paddle_driver/initialize_paddle_driver.py +++ b/fastNLP/core/drivers/paddle_driver/initialize_paddle_driver.py @@ -14,20 +14,24 @@ from fastNLP.core.log import logger if _NEED_IMPORT_PADDLE: import paddle +__all__ = [] + def initialize_paddle_driver(driver: str, device: Optional[Union[str, int, List[int]]], model: "paddle.nn.Layer", **kwargs) -> PaddleDriver: r""" - 用来根据参数 `driver` 和 `device` 来确定并且初始化一个具体的 `Driver` 实例然后返回回去； - 1、如果检测到当前进程为用户通过 `python -m paddle.distributed.launch xxx.py` 方式拉起的，则将 - 设备自动设置为用户指定的设备（由于我们在引入 fastNLP 进行了特殊的设置，因此可以通过 `CUDA_VISIBLE_DEVICES` 获取） - 2、如果检测到输入的 `driver` 是 `paddle` 但 `device` 包含了多个设备，那么我们会给出警告并且自动返回多卡的 Driver - 3、如果检测到输入的 `driver` 是 `fleet` 但 `device` 仅有一个设备，那么我们会给出警告但仍旧返回多卡的 Driver + 用来根据参数 ``device`` 来确定并且初始化一个具体的 ``Driver`` 实例。 + + 1. 如果检测到当前进程为用户通过 ``python -m paddle.distributed.launch xxx.py`` 方式拉起的，则将 + 设备自动设置为用户指定的设备（由于我们要求分布式训练必须进行 ``backend`` 的设置，因此可以通过 ``CUDA_VISIBLE_DEVICES`` 获取） + + 2. 如果 ``device`` 包含了多个设备，则返回一个 :class:`~fastNLP.core.PaddleFleetDriver` 实例，否则返回 + 单卡的 :class:`~fastNLP.core.PaddleSingleDriver` 实例 - :param driver: 使用的 ``driver`` 类型，在这个函数中仅支持 ``paddle`` - :param device: 该参数的格式与 `Trainer` 对参数 `device` 的要求一致； + :param driver: 使用的 ``driver`` 类型，在这个函数中仅支持 ``paddle``； + :param device: 该参数的格式与 ``Trainer`` 对参数 ``device`` 的要求一致； :param model: 训练或者评测的具体的模型； - :return: 返回构造的 `Driver` 实例。 + :return: 一个 :class:`~fastNLP.core.PaddleSingleDriver` 或 :class:`~fastNLP.core.PaddleFleetDriver` 实例； """ if driver != "paddle": raise ValueError("When initialize PaddleDriver, parameter `driver` must be 'paddle'.") diff --git a/fastNLP/core/drivers/paddle_driver/paddle_driver.py b/fastNLP/core/drivers/paddle_driver/paddle_driver.py index cf35af3a..74c7b7a8 100644 --- a/fastNLP/core/drivers/paddle_driver/paddle_driver.py +++ b/fastNLP/core/drivers/paddle_driver/paddle_driver.py @@ -12,7 +12,8 @@ from fastNLP.envs.env import USER_CUDA_VISIBLE_DEVICES from .utils import _build_fp16_env, optimizer_state_to_device, DummyGradScaler from fastNLP.envs.imports import _NEED_IMPORT_PADDLE from fastNLP.core.drivers.driver import Driver -from fastNLP.core.utils import apply_to_collection, paddle_move_data_to_device, get_device_from_visible +from fastNLP.core.utils import apply_to_collection, paddle_move_data_to_device +from fastNLP.core.utils.paddle_utils import _convert_data_device from fastNLP.envs import ( FASTNLP_SEED_WORKERS, FASTNLP_MODEL_FILENAME, @@ -371,10 +372,7 @@ class PaddleDriver(Driver): :return: 将移动到指定机器上的 batch 对象返回； """ - if USER_CUDA_VISIBLE_DEVICES in os.environ: - device = get_device_from_visible(self.data_device) - else: - device = self.data_device + device = _convert_data_device(self.data_device) return paddle_move_data_to_device(batch, device) @staticmethod diff --git a/fastNLP/core/drivers/paddle_driver/single_device.py b/fastNLP/core/drivers/paddle_driver/single_device.py index 6d553fea..c0957dbf 100644 --- a/fastNLP/core/drivers/paddle_driver/single_device.py +++ b/fastNLP/core/drivers/paddle_driver/single_device.py @@ -8,10 +8,10 @@ from fastNLP.envs.imports import _NEED_IMPORT_PADDLE from fastNLP.envs.env import USER_CUDA_VISIBLE_DEVICES from fastNLP.core.utils import ( auto_param_call, - get_device_from_visible, get_paddle_gpu_str, get_paddle_device_id, ) +from fastNLP.core.utils.paddle_utils import _convert_data_device from fastNLP.core.utils.utils import _get_fun_msg from fastNLP.core.samplers import ( ReproducibleBatchSampler, @@ -64,10 +64,7 @@ class PaddleSingleDriver(PaddleDriver): r""" 该函数用来初始化训练环境，用于设置当前训练的设备，并将模型迁移到对应设备上。 """ - if USER_CUDA_VISIBLE_DEVICES in os.environ: - device = get_device_from_visible(self.data_device) - else: - device = self.data_device + device = _convert_data_device(self.data_device) paddle.device.set_device(device) with contextlib.redirect_stdout(None): diff --git a/fastNLP/core/drivers/torch_driver/initialize_torch_driver.py b/fastNLP/core/drivers/torch_driver/initialize_torch_driver.py index 025744bb..f9fac83f 100644 --- a/fastNLP/core/drivers/torch_driver/initialize_torch_driver.py +++ b/fastNLP/core/drivers/torch_driver/initialize_torch_driver.py @@ -10,19 +10,18 @@ from .ddp import TorchDDPDriver from fastNLP.core.log import logger from fastNLP.envs import FASTNLP_BACKEND_LAUNCH +__all__ = [] def initialize_torch_driver(driver: str, device: Optional[Union[str, "torch.device", int, List[int]]], model: "torch.nn.Module", **kwargs) -> TorchDriver: r""" - 用来根据参数 `driver` 和 `device` 来确定并且初始化一个具体的 `Driver` 实例然后返回回去； - 注意如果输入的 `device` 如果和 `driver` 对应不上就直接报错； + 用来根据参数 ``driver` 和 ``device`` 来确定并且初始化一个具体的 ``Driver`` 实例然后返回回去； - :param driver: 该参数的值应为以下之一：["torch", "torch_ddp", "fairscale"]； - :param device: 该参数的格式与 `Trainer` 对参数 `device` 的要求一致； + :param driver: 该参数的值应为以下之一：``["torch", "fairscale"]``； + :param device: 该参数的格式与 ``Trainer`` 对参数 ``device`` 的要求一致； :param model: 训练或者评测的具体的模型； - :return: 返回一个元组，元组的第一个值是具体的基于 pytorch 的 `Driver` 实例，元组的第二个值是该 driver 的名字（用于检测一个脚本中 - 先后 driver 的次序的正确问题）； + :return: 返回一个 :class:`~fastNLP.core.TorchSingleDriver` 或 :class:`~fastNLP.core.TorchDDPDriver` 实例； """ # world_size 和 rank if FASTNLP_BACKEND_LAUNCH in os.environ: diff --git a/fastNLP/core/metrics/backend/paddle_backend/backend.py b/fastNLP/core/metrics/backend/paddle_backend/backend.py index 74cf6b82..b8ea9cb0 100644 --- a/fastNLP/core/metrics/backend/paddle_backend/backend.py +++ b/fastNLP/core/metrics/backend/paddle_backend/backend.py @@ -4,7 +4,7 @@ from typing import List, Any import numpy as np from fastNLP.core.metrics.backend import Backend -from fastNLP.core.utils.paddle_utils import paddle_to, get_device_from_visible +from fastNLP.core.utils.paddle_utils import paddle_to, _convert_data_device from fastNLP.core.metrics.utils import AggregateMethodError from fastNLP.core.drivers.paddle_driver.dist_utils import fastnlp_paddle_all_gather from fastNLP.envs.imports import _NEED_IMPORT_PADDLE @@ -81,8 +81,7 @@ class PaddleBackend(Backend): raise ValueError(f"tensor: {tensor} can not convert to ndarray!") def move_tensor_to_device(self, tensor, device): - if USER_CUDA_VISIBLE_DEVICES in os.environ: - device = get_device_from_visible(device) + device = _convert_data_device(device) return paddle_to(tensor, device) def all_gather_object(self, obj, group=None) -> List: diff --git a/fastNLP/core/utils/__init__.py b/fastNLP/core/utils/__init__.py index aca01344..6c65c8a5 100644 --- a/fastNLP/core/utils/__init__.py +++ b/fastNLP/core/utils/__init__.py @@ -2,7 +2,6 @@ __all__ = [ 'cache_results', 'is_jittor_dataset', 'jittor_collate_wraps', - 'get_device_from_visible', 'paddle_to', 'paddle_move_data_to_device', 'get_paddle_device_id', @@ -28,7 +27,7 @@ __all__ = [ from .cache_results import cache_results from .jittor_utils import is_jittor_dataset, jittor_collate_wraps -from .paddle_utils import get_device_from_visible, paddle_to, paddle_move_data_to_device, get_paddle_device_id, get_paddle_gpu_str, is_in_paddle_dist, \ +from .paddle_utils import paddle_to, paddle_move_data_to_device, get_paddle_device_id, get_paddle_gpu_str, is_in_paddle_dist, \ is_in_fnlp_paddle_dist, is_in_paddle_launch_dist from .rich_progress import f_rich_progress from .torch_utils import torch_move_data_to_device diff --git a/fastNLP/core/utils/jittor_utils.py b/fastNLP/core/utils/jittor_utils.py index 89686cff..08b3b7a8 100644 --- a/fastNLP/core/utils/jittor_utils.py +++ b/fastNLP/core/utils/jittor_utils.py @@ -15,6 +15,12 @@ from fastNLP.core.dataset import Instance def is_jittor_dataset(dataset) -> bool: + """ + 判断传入的 ``dataset`` 是否是 :class:`jittor.dataset.Dataset` 类型 + + :param dataset: 数据集； + :return: 当前 ``dataset`` 是否为 ``jittor`` 的数据集类型； + """ try: if isinstance(dataset, jt.dataset.Dataset): return True @@ -26,7 +32,8 @@ def is_jittor_dataset(dataset) -> bool: def jittor_collate_wraps(func, auto_collator: Callable): """ - 对jittor的collate_fn进行wrap封装, 如果数据集为mapping类型，那么采用auto_collator，否则还是采用jittor自带的collate_batch + 对 ``jittor`` 的 ``collate_fn`` 进行 ``wrap`` 封装,。如果数据集为 ``mapping`` 类型，那么采用 ``auto_collator`` ，否则 + 还是采用 ``jittor`` 的 ``collate_batch``。 :param func: :param auto_collator: diff --git a/fastNLP/core/utils/paddle_utils.py b/fastNLP/core/utils/paddle_utils.py index 13fe7b07..c7bb9e79 100644 --- a/fastNLP/core/utils/paddle_utils.py +++ b/fastNLP/core/utils/paddle_utils.py @@ -1,5 +1,4 @@ __all__ = [ - "get_device_from_visible", "paddle_to", "paddle_move_data_to_device", "get_paddle_gpu_str", @@ -21,55 +20,71 @@ if _NEED_IMPORT_PADDLE: from .utils import apply_to_collection -def get_device_from_visible(device: Union[str, int]) -> str: +def _convert_data_device(device: Union[str, int]) -> str: """ - 在有 ``CUDA_VISIBLE_DEVICES`` 的情况下，获取对应的设备。 - 如 CUDA_VISIBLE_DEVICES=2,3 ，device=3 ，则返回1。 + 用于转换 ``driver`` 的 ``data_device`` 的函数。如果用户设置了 ``FASTNLP_BACKEND=paddle``，那么 ``fastNLP`` 会将 + 可见的设备保存在 ``USER_CUDA_VISIBLE_DEVICES`` 中，并且将 ``CUDA_VISIBLE_DEVICES`` 设置为可见的第一张显卡；这是为 + 了顺利执行 ``paddle`` 的分布式训练而设置的。 + + 在这种情况下，单纯使用 ``driver.data_device`` 是无效的。比如在分布式训练中将设备设置为 ``[0,2,3]`` ，且用户设置了 + ``CUDA_VISIBLE_DEVICES=3,4,5,6`` ，那么在 ``rank1``的进程中有:: - :param device: 未转化的设备名 - :return: 转化后的设备，格式为 ``gpu:x`` + os.environ["CUDA_VISIBLE_DEVICES"] = "5" + os.environ["USER_CUDA_VISIBLE_DEVICES"] = "3,4,5,6" + driver.data_device = "gpu:2" # 为了向用户正确地反映他们设置的设备减少歧义，因此这里没有设置为 "gpu:5" + + 此时我们便需要通过这个函数将 ``data_device`` 转换为 ``gpu:0``。具体过程便是通过索引 **2** 在 ``USER_CUDA_VISIBLE_DEVICES`` 中 + 找到设备 **5**，然后在 ``CUDA_VISIBLE_DEVICES`` 中找到设备 **5** 的索引 **0** 返回。 + + .. note:: + + 在分布式单进程仅支持单卡的情况下中，这个函数实际等同于直接转换为 ``gpu:0`` 返回。 + + :param device: 未转化的设备； + :return: 转化后的设备，格式为 ``gpu:x``； """ - if device == "cpu": - return device - cuda_visible_devices = os.getenv("CUDA_VISIBLE_DEVICES") - user_visible_devices = os.getenv(USER_CUDA_VISIBLE_DEVICES) - if cuda_visible_devices is not None: + try: + user_visible_devices = os.getenv(USER_CUDA_VISIBLE_DEVICES) + if device == "cpu" or user_visible_devices is None: + # 传入的是 CPU，或者没有设置 USER_CUDA_VISIBLE_DEVICES + # 此时不需要进行转换 + return get_paddle_gpu_str(device) + idx = get_paddle_device_id(device) - if user_visible_devices is not None: - # 此时一定发生在分布式的情况下，利用 USER_CUDA_VISIBLDE_DEVICES 获取用户期望的设备 - idx = user_visible_devices.split(",")[idx] - else: - idx = str(idx) - - cuda_visible_devices_list = cuda_visible_devices.split(',') - if idx not in cuda_visible_devices_list: - raise ValueError(f"Can't find your devices {idx} in CUDA_VISIBLE_DEVICES[{cuda_visible_devices}]. ") + idx = user_visible_devices.split(",")[idx] + # 此时 CUDA_VISIBLE_DEVICES 一定不是 None + cuda_visible_devices_list = os.getenv("CUDA_VISIBLE_DEVICES").split(',') return f"gpu:{cuda_visible_devices_list.index(idx)}" - else: - return get_paddle_gpu_str(device) + except Exception as e: + raise ValueError(f"Can't convert device {device} when USER_CUDA_VISIBLE_DEVICES={user_visible_devices} " + "and CUDA_VISIBLE_DEVICES={cuda_visible_devices}. If this situation happens, please report this bug to us.") -def paddle_to(data, device: Union[str, int]): +def paddle_to(data: "paddle.Tensor", device: Union[str, int]) -> "paddle.Tensor": """ - 将 `data` 迁移到指定的 `device` 上 + 将 ``data`` 迁移到指定的 ``device`` 上。``paddle.Tensor`` 没有类似 ``torch.Tensor`` 的 ``to`` 函数，该函数 + 只是集成了 :func:`paddle.Tensor.cpu` 和 :func:`paddle.Tensor.cuda` 两个函数。 - :param data: 要迁移的张量 - :param device: 目标设备，可以是 `str` 或 `int` - :return: 迁移后的张量 + :param data: 要迁移的张量； + :param device: 目标设备，可以是 ``str`` 或 ``int`` 类型； + :return: 迁移后的张量； """ if device == "cpu": return data.cpu() else: - # device = get_device_from_visible(device, output_type=int) return data.cuda(get_paddle_device_id(device)) - def get_paddle_gpu_str(device: Union[str, int]) -> str: """ - 获得 `gpu:x` 类型的设备名 + 获得 ``gpu:x`` 格式的设备名:: - :param device: 设备编号或设备名 - :return: 返回对应的 `gpu:x` 格式的设备名 + >>> get_paddle_gpu_str(1) + 'gpu:1' + >>> get_paddle_gpu_str("cuda:1") + 'gpu:1' + + :param device: 设备编号或设备名； + :return: 返回对应的 ``gpu:x`` 格式的设备名； """ if isinstance(device, str): return device.replace("cuda", "gpu") @@ -78,10 +93,17 @@ def get_paddle_gpu_str(device: Union[str, int]) -> str: def get_paddle_device_id(device: Union[str, int]) -> int: """ - 获得 gpu 的设备id + 获得 ``device`` 的设备编号:: + + >>> get_paddle_device_id("gpu:1") + 1 + >>> get_paddle_device_id("gpu") + 0 + + 请注意不要向这个函数中传入 ``cpu``。 - :param: device: 设备编号或设备名 - :return: 设备对应的编号 + :param: device: 设备编号或设备名； + :return: 设备对应的编号； """ if isinstance(device, int): return device @@ -103,21 +125,17 @@ def get_paddle_device_id(device: Union[str, int]) -> int: return device_id -def paddle_move_data_to_device(batch: Any, device: Optional[str] = None, - data_device: Optional[str] = None) -> Any: +def paddle_move_data_to_device(batch: Any, device: Optional[Union[str, int]]) -> Any: r""" - 将数据集合传输到给定设备。只有paddle.Tensor对象会被传输到设备中，其余保持不变 + 将 ``paddle`` 的数据集合传输到给定设备。只有 :class:`paddle.Tensor` 对象会被传输到设备中，其余保持不变。 - :param batch: - :param device: `cpu`, `gpu` or `gpu:x` - :param data_device: - :return: 相同的集合，但所有包含的张量都驻留在新设备上； + :param batch: 需要进行迁移的数据集合； + :param device: 目标设备。可以是显卡设备的编号，或是``cpu``, ``gpu`` 或 ``gpu:x`` 格式的字符串；当这个参数 + 为 `None`` 时，不会执行任何操作。 + :return: 迁移到新设备上的数据集合； """ if device is None: - if data_device is not None: - device = data_device - else: - return batch + return batch def batch_to(data: Any) -> Any: return paddle_to(data, device) @@ -125,22 +143,22 @@ def paddle_move_data_to_device(batch: Any, device: Optional[str] = None, return apply_to_collection(batch, dtype=paddle.Tensor, function=batch_to) -def is_in_paddle_dist(): +def is_in_paddle_dist() -> bool: """ - 判断是否处于分布式的进程下，使用 global_rank 和 selected_gpus 判断 + 判断是否处于 ``paddle`` 分布式的进程下，使用 ``PADDLE_RANK_IN_NODE`` 和 ``FLAGS_selected_gpus`` 判断。 """ return ('PADDLE_RANK_IN_NODE' in os.environ and 'FLAGS_selected_gpus' in os.environ) -def is_in_fnlp_paddle_dist(): +def is_in_fnlp_paddle_dist() -> bool: """ - 判断是否处于 FastNLP 拉起的分布式进程中 + 判断是否处于 ``fastNLP`` 拉起的 ``paddle`` 分布式进程中 """ return FASTNLP_DISTRIBUTED_CHECK in os.environ -def is_in_paddle_launch_dist(): +def is_in_paddle_launch_dist() -> bool: """ - 判断是否处于 launch 启动的分布式进程中 + 判断是否处于 ``python -m paddle.distributed.launch`` 方法启动的 ``paddle`` 分布式进程中 """ return FASTNLP_BACKEND_LAUNCH in os.environ \ No newline at end of file diff --git a/fastNLP/core/utils/torch_utils.py b/fastNLP/core/utils/torch_utils.py index 72f1058f..862ea20d 100644 --- a/fastNLP/core/utils/torch_utils.py +++ b/fastNLP/core/utils/torch_utils.py @@ -44,12 +44,12 @@ class TorchTransferableDataType(ABC): def torch_move_data_to_device(batch: Any, device: Optional[Union[str, "torch.device"]] = None, non_blocking: Optional[bool] = True) -> Any: r""" - 将数据集合传输到给定设备。任何定义方法 “to(device)” 的对象都将被移动并且集合中的所有其他对象将保持不变； + 在 ``pytorch`` 中将数据集合 ``batch`` 传输到给定设备。任何定义方法 ``to(device)`` 的对象都将被移动并且集合中的所有其他对象将保持不变； - :param batch: 应当迁移的数据； - :param device: 数据应当迁移到的设备；当该参数的值为 None 时，表示迁移数据的操作由用户自己完成，我们不需要经管； - :param non_blocking: pytorch 的迁移数据方法 `to` 的参数； - :return: 相同的集合，但所有包含的张量都驻留在新设备上； + :param batch: 需要迁移的数据； + :param device: 数据应当迁移到的设备；当该参数的值为 ``None`` 时则不执行任何操作； + :param non_blocking: ``pytorch`` 的数据迁移方法 ``to`` 的参数； + :return: 迁移到新设备上的数据集合； """ if device is None: return batch diff --git a/fastNLP/core/utils/utils.py b/fastNLP/core/utils/utils.py index ec7a8b47..00da9ac1 100644 --- a/fastNLP/core/utils/utils.py +++ b/fastNLP/core/utils/utils.py @@ -38,10 +38,16 @@ __all__ = [ def get_fn_arg_names(fn: Callable) -> List[str]: r""" - 返回一个函数所有参数的名字 + 该函数可以返回一个函数所有参数的名字:: - :param fn: 需要查询的函数 - :return: 一个列表，其中的元素是函数 ``fn`` 参数的字符串名字 + >>> def function(a, b=1): + ... return a + ... + >>> get_fn_arg_names(function) + ['a', 'b'] + + :param fn: 需要查询的函数； + :return: 包含函数 ``fn`` 参数名的列表； """ return list(inspect.signature(fn).parameters) @@ -49,7 +55,7 @@ def get_fn_arg_names(fn: Callable) -> List[str]: def auto_param_call(fn: Callable, *args, signature_fn: Optional[Callable] = None, mapping: Optional[Dict[AnyStr, AnyStr]] = None) -> Any: r""" - 该函数会根据输入函数的形参名从 ``*args`` （因此都需要是 ``dict`` 类型）中找到匹配的值进行调用，如果传入的数据与 ``fn`` 的形参不匹配，可以通过 + 该函数会根据输入函数的形参名从 ``*args`` （均为 ``dict`` 类型）中找到匹配的值进行调用，如果传入的数据与 ``fn`` 的形参不匹配，可以通过 ``mapping`` 参数进行转换。``mapping`` 参数中的一对 ``(key, value)`` 表示在 ``*args`` 中找到 ``key`` 对应的值，并将这个值传递给形参中名为 ``value`` 的参数。 @@ -161,13 +167,13 @@ def _get_keys(args:List[Dict]) -> List[List[str]]: def _get_fun_msg(fn, with_fp=True)->str: """ - 获取函数的基本信息，帮助报错。 - ex: - print(_get_fun_msg(_get_fun_msg)) - # `_get_fun_msg(fn) -> str`(In file:/Users/hnyan/Desktop/projects/fastNLP/fastNLP/fastNLP/core/utils/utils.py) + 获取函数的基本信息，帮助报错:: + + >>>> print(_get_fun_msg(_get_fun_msg)) + `_get_fun_msg(fn) -> str`(In file:/Users/hnyan/Desktop/projects/fastNLP/fastNLP/fastNLP/core/utils/utils.py) :param callable fn: - :param with_fp: 是否包含函数所在的文件信息。 + :param with_fp: 是否包含函数所在的文件信息； :return: """ if isinstance(fn, functools.partial): @@ -224,7 +230,7 @@ def _check_valid_parameters_number(fn, expected_params:List[str], fn_name=None): def check_user_specific_params(user_params: Dict, fn: Callable): """ 该函数使用用户的输入来对指定函数的参数进行赋值，主要用于一些用户无法直接调用函数的情况； - 该函数主要的作用在于帮助检查用户对使用函数 ``fn`` 的参数输入是否有误； + 主要作用在于帮助检查用户对使用函数 ``fn`` 的参数输入是否有误； :param user_params: 用户指定的参数的值，应当是一个字典，其中 ``key`` 表示每一个参数的名字， ``value`` 为每一个参数的值； @@ -241,7 +247,7 @@ def check_user_specific_params(user_params: Dict, fn: Callable): def dataclass_to_dict(data: "dataclasses.dataclass") -> Dict: """ - 将传入的 `dataclass` 实例转换为字典。 + 将传入的 ``dataclass`` 实例转换为字典。 """ if not is_dataclass(data): raise TypeError(f"Parameter `data` can only be `dataclass` type instead of {type(data)}.") @@ -253,12 +259,12 @@ def dataclass_to_dict(data: "dataclasses.dataclass") -> Dict: def match_and_substitute_params(mapping: Optional[Union[Callable, Dict]] = None, data: Optional[Any] = None) -> Any: r""" - 用来实现将输入的 ``batch``，或者输出的 ``outputs``，通过 ``mapping`` 将键值进行更换的功能； + 用来实现将输入的 ``batch`` 或者输出的 ``outputs`` 通过 ``mapping`` 将键值进行更换的功能； 该函数应用于 ``input_mapping`` 和 ``output_mapping``； - 对于 ``input_mapping``，该函数会在 :class:`~fastNLP.core.controllers.TrainBatchLoop` 中取完数据后立刻被调用； - 对于 ``output_mapping``，该函数会在 :class:`~fastNLP.core.Trainer` 的 :meth:`~fastNLP.core.Trainer.train_step` - 以及 :class:`~fastNLP.core.Evaluator` 的 :meth:`~fastNLP.core.Evaluator.train_step` 中得到结果后立刻被调用； + * 对于 ``input_mapping``，该函数会在 :class:`~fastNLP.core.controllers.TrainBatchLoop` 中取完数据后立刻被调用； + * 对于 ``output_mapping``，该函数会在 :class:`~fastNLP.core.Trainer` 的 :meth:`~fastNLP.core.Trainer.train_step` + 以及 :class:`~fastNLP.core.Evaluator` 的 :meth:`~fastNLP.core.Evaluator.train_step` 中得到结果后立刻被调用； 转换的逻辑按优先级依次为： @@ -277,9 +283,9 @@ def match_and_substitute_params(mapping: Optional[Union[Callable, Dict]] = None, 然后使用 ``mapping`` 对这个 ``Dict`` 进行转换，如果没有匹配上 ``mapping`` 中的 ``key`` 则保持 ``\'\_number\'`` 这个形式。 - :param mapping: 用于转换的字典或者函数；``mapping`` 是函数时，返回值必须为字典类型。 + :param mapping: 用于转换的字典或者函数；当 ``mapping`` 是函数时，返回值必须为字典类型； :param data: 需要被转换的对象； - :return: 返回转换好的结果； + :return: 返回转换后的结果； """ if mapping is None: return data @@ -331,19 +337,19 @@ def apply_to_collection( **kwargs: Any, ) -> Any: """ - 使用函数 ``function`` 递归地在 ``data`` 中的元素执行，但是仅在满足元素为 ``dtype`` 时执行。 + 递归地对 ``data`` 中的元素执行函数 ``function``，且仅在满足元素为 ``dtype`` 时执行。 该函数参考了 `pytorch-lightning `_ 的实现 - :param data: 需要进行处理的数据集合或数据 - :param dtype: 数据的类型，函数 ``function`` 只会被应用于 ``data`` 中类型为 ``dtype`` 的数据 - :param function: 对数据进行处理的函数 - :param args: ``function`` 所需要的其它参数 + :param data: 需要进行处理的数据集合或数据； + :param dtype: 数据的类型，函数 ``function`` 只会被应用于 ``data`` 中类型为 ``dtype`` 的数据； + :param function: 对数据进行处理的函数； + :param args: ``function`` 所需要的其它参数； :param wrong_dtype: ``function`` 一定不会生效的数据类型。如果数据既是 ``wrong_dtype`` 类型又是 ``dtype`` 类型 - 那么也不会生效。 - :param include_none: 是否包含执行结果为 ``None`` 的数据，默认为 ``True``。 - :param kwargs: ``function`` 所需要的其它参数 - :return: 经过 ``function`` 处理后的数据集合 + 那么也不会生效； + :param include_none: 是否包含执行结果为 ``None`` 的数据，默认为 ``True``； + :param kwargs: ``function`` 所需要的其它参数； + :return: 经过 ``function`` 处理后的数据集合； """ # Breaking condition if isinstance(data, dtype) and (wrong_dtype is None or not isinstance(data, wrong_dtype)): @@ -411,20 +417,20 @@ def apply_to_collection( @contextmanager def nullcontext(): r""" - 实现一个什么都不做的上下文环境 + 实现一个什么都不做的上下文环境。 """ yield def sub_column(string: str, c: int, c_size: int, title: str) -> str: r""" - 对传入的字符串进行截断，方便在命令行中显示 + 对传入的字符串进行截断，方便在命令行中显示。 - :param string: 要被截断的字符串 - :param c: 命令行列数 - :param c_size: :class:`~fastNLP.core.Instance` 或 :class:`fastNLP.core.DataSet` 的 ``field`` 数目 - :param title: 列名 - :return: 对一个过长的列进行截断的结果 + :param string: 要被截断的字符串； + :param c: 命令行列数； + :param c_size: :class:`~fastNLP.core.Instance` 或 :class:`fastNLP.core.DataSet` 的 ``field`` 数目； + :param title: 列名； + :return: 对一个过长的列进行截断的结果； """ avg = max(int(c / c_size / 2), len(title)) string = str(string) @@ -453,7 +459,7 @@ def _is_iterable(value): def pretty_table_printer(dataset_or_ins) -> PrettyTable: r""" - 在 ``fastNLP`` 中展示数据的函数:: + 用于在 ``fastNLP`` 中展示数据的函数:: >>> ins = Instance(field_1=[1, 1, 1], field_2=[2, 2, 2], field_3=["a", "b", "c"]) +-----------+-----------+-----------------+ @@ -462,8 +468,8 @@ def pretty_table_printer(dataset_or_ins) -> PrettyTable: | [1, 1, 1] | [2, 2, 2] | ['a', 'b', 'c'] | +-----------+-----------+-----------------+ - :param dataset_or_ins: 要展示的 :class:`~fastNLP.core.DataSet` 或者 :class:`~fastNLP.core.Instance` - :return: 根据 ``terminal`` 大小进行自动截断的数据表格 + :param dataset_or_ins: 要展示的 :class:`~fastNLP.core.DataSet` 或者 :class:`~fastNLP.core.Instance` 实例； + :return: 根据命令行大小进行自动截断的数据表格； """ x = PrettyTable() try: @@ -529,7 +535,7 @@ def deprecated(help_message: Optional[str] = None): """ 标记当前功能已经过时的装饰器。 - :param help_message: 一段指引信息，告知用户如何将代码切换为当前版本提倡的用法。 + :param help_message: 一段指引信息，告知用户如何将代码切换为当前版本提倡的用法； """ def decorator(deprecated_function: Callable): @@ -578,10 +584,10 @@ def seq_len_to_mask(seq_len, max_len: Optional[int]): >>>print(mask.size()) torch.Size([14, 100]) - :param seq_len: 大小为是 ``(B,)`` 的长度序列 - :param int max_len: 将长度 ``pad`` 到 ``max_len``。默认情况（为 ``None``）使用的是 ``seq_len`` 中最长的长度。 + :param seq_len: 大小为 ``(B,)`` 的长度序列； + :param int max_len: 将长度补齐或截断到 ``max_len``。默认情况（为 ``None``）使用的是 ``seq_len`` 中最长的长度； 但在 :class:`torch.nn.DataParallel` 等分布式的场景下可能不同卡的 ``seq_len`` 会有区别，所以需要传入 - 一个 ``max_len`` 使得 ``mask`` 的长度 ``pad`` 到该长度。 + 一个 ``max_len`` 使得 ``mask`` 的补齐或截断到该长度。 :return: 大小为 ``(B, max_len)`` 的 ``mask``， 元素类型为 ``bool`` 或 ``uint8`` """ if isinstance(seq_len, np.ndarray): diff --git a/tests/core/controllers/_test_trainer_fleet_outside.py b/tests/core/controllers/_test_trainer_fleet_outside.py index f5b7fc4d..963276db 100644 --- a/tests/core/controllers/_test_trainer_fleet_outside.py +++ b/tests/core/controllers/_test_trainer_fleet_outside.py @@ -67,7 +67,6 @@ def test_trainer_fleet( validate_dataloaders = val_dataloader validate_every = MNISTTrainFleetConfig.validate_every metrics = {"acc": Accuracy()} - data_device = f'gpu:{os.environ["USER_CUDA_VISIBLE_DEVICES"].split(",").index(os.environ["CUDA_VISIBLE_DEVICES"])}' trainer = Trainer( model=model, driver=driver, @@ -83,7 +82,6 @@ def test_trainer_fleet( n_epochs=n_epochs, callbacks=callbacks, # output_from_new_proc="logs", - data_device=data_device ) trainer.run() diff --git a/tests/core/utils/test_paddle_utils.py b/tests/core/utils/test_paddle_utils.py index 96a3b41a..c5daac63 100644 --- a/tests/core/utils/test_paddle_utils.py +++ b/tests/core/utils/test_paddle_utils.py @@ -2,7 +2,7 @@ import os import pytest -from fastNLP.core.utils.paddle_utils import get_device_from_visible, paddle_to, paddle_move_data_to_device +from fastNLP.core.utils.paddle_utils import _convert_data_device, paddle_to, paddle_move_data_to_device from fastNLP.envs.imports import _NEED_IMPORT_PADDLE if _NEED_IMPORT_PADDLE: import paddle @@ -11,24 +11,24 @@ if _NEED_IMPORT_PADDLE: ("user_visible_devices, cuda_visible_devices, device, correct"), ( (None, None, 1, "gpu:1"), - (None, "2,4,5,6", 5, "gpu:2"), - (None, "3,4,5", 4, "gpu:1"), - ("0,1,2,3,4,5,6,7", "0", "cpu", "cpu"), + (None, "2,4,5,6", 2, "gpu:2"), + (None, "3,4,5", 1, "gpu:1"), ("0,1,2,3,4,5,6,7", "0", "cpu", "cpu"), + ("3,4,5,6,7", "0", "cpu", "cpu"), ("0,1,2,3,4,5,6,7", "3,4,5", "gpu:4", "gpu:1"), ("0,1,2,3,4,5,6,7", "3,4,5", "gpu:5", "gpu:2"), ("3,4,5,6", "3,5", 0, "gpu:0"), ("3,6,7,8", "6,7,8", "gpu:2", "gpu:1"), ) ) -def test_get_device_from_visible(user_visible_devices, cuda_visible_devices, device, correct): +def test_convert_data_device(user_visible_devices, cuda_visible_devices, device, correct): _cuda_visible_devices = os.getenv("CUDA_VISIBLE_DEVICES") _user_visible_devices = os.getenv("USER_CUDA_VISIBLE_DEVICES") if cuda_visible_devices is not None: os.environ["CUDA_VISIBLE_DEVICES"] = cuda_visible_devices if user_visible_devices is not None: os.environ["USER_CUDA_VISIBLE_DEVICES"] = user_visible_devices - res = get_device_from_visible(device) + res = _convert_data_device(device) assert res == correct # 还原环境变量 From 3533bc27044f11bfe4a8d9be64f37d7f51716301 Mon Sep 17 00:00:00 2001 From: x54-729 <17307130121@fudan.edu.cn> Date: Tue, 10 May 2022 11:35:41 +0000 Subject: [PATCH 14/15] =?UTF-8?q?=E6=81=A2=E5=A4=8D=20torch=20paddle=20jit?= =?UTF-8?q?tor=20=E4=B9=8B=E9=97=B4=E7=9A=84=E8=BD=AC=E6=8D=A2=E5=87=BD?= =?UTF-8?q?=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/modules/__init__.py | 9 + fastNLP/modules/mix_modules/__init__.py | 10 + fastNLP/modules/mix_modules/utils.py | 0 tests/modules/__init__.py | 0 tests/modules/mix_modules/__init__.py | 0 tests/modules/mix_modules/test_utils.py | 442 ++++++++++++++++++++++++ 6 files changed, 461 insertions(+) create mode 100644 fastNLP/modules/__init__.py create mode 100644 fastNLP/modules/mix_modules/__init__.py create mode 100644 fastNLP/modules/mix_modules/utils.py create mode 100644 tests/modules/__init__.py create mode 100644 tests/modules/mix_modules/__init__.py create mode 100644 tests/modules/mix_modules/test_utils.py diff --git a/fastNLP/modules/__init__.py b/fastNLP/modules/__init__.py new file mode 100644 index 00000000..db7c9436 --- /dev/null +++ b/fastNLP/modules/__init__.py @@ -0,0 +1,9 @@ +__all__ = [ + # "MixModule", + "torch2paddle", + "paddle2torch", + "torch2jittor", + "jittor2torch", +] + +from .mix_modules import torch2paddle, paddle2torch, torch2jittor, jittor2torch \ No newline at end of file diff --git a/fastNLP/modules/mix_modules/__init__.py b/fastNLP/modules/mix_modules/__init__.py new file mode 100644 index 00000000..bd8b4e8f --- /dev/null +++ b/fastNLP/modules/mix_modules/__init__.py @@ -0,0 +1,10 @@ +__all__ = [ + # "MixModule", + "torch2paddle", + "paddle2torch", + "torch2jittor", + "jittor2torch", +] + +# from .mix_module import MixModule +from .utils import * \ No newline at end of file diff --git a/fastNLP/modules/mix_modules/utils.py b/fastNLP/modules/mix_modules/utils.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/modules/__init__.py b/tests/modules/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/modules/mix_modules/__init__.py b/tests/modules/mix_modules/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/modules/mix_modules/test_utils.py b/tests/modules/mix_modules/test_utils.py new file mode 100644 index 00000000..890a714a --- /dev/null +++ b/tests/modules/mix_modules/test_utils.py @@ -0,0 +1,442 @@ +import pytest + +from fastNLP.envs.imports import _NEED_IMPORT_JITTOR, _NEED_IMPORT_PADDLE, _NEED_IMPORT_TORCH +from fastNLP.modules.mix_modules.utils import ( + paddle2torch, + torch2paddle, + jittor2torch, + torch2jittor, +) + +if _NEED_IMPORT_TORCH: + import torch + +if _NEED_IMPORT_PADDLE: + import paddle + +if _NEED_IMPORT_JITTOR: + import jittor + + +############################################################################ +# +# 测试paddle到torch的转换 +# +############################################################################ + +@pytest.mark.torchpaddle +class TestPaddle2Torch: + + def check_torch_tensor(self, tensor, device, requires_grad): + """ + 检查张量设备和梯度情况的工具函数 + """ + + assert isinstance(tensor, torch.Tensor) + assert tensor.device == torch.device(device) + assert tensor.requires_grad == requires_grad + + def test_gradient(self): + """ + 测试张量转换后的反向传播是否正确 + """ + + x = paddle.to_tensor([1.0, 2.0, 3.0, 4.0, 5.0], stop_gradient=False) + y = paddle2torch(x) + z = 3 * (y ** 2) + z.sum().backward() + assert y.grad.tolist() == [6, 12, 18, 24, 30] + + def test_tensor_transfer(self): + """ + 测试单个张量的设备和梯度转换是否正确 + """ + + paddle_tensor = paddle.rand((3, 4, 5)).cpu() + res = paddle2torch(paddle_tensor) + self.check_torch_tensor(res, "cpu", not paddle_tensor.stop_gradient) + + res = paddle2torch(paddle_tensor, target_device="cuda:2", no_gradient=None) + self.check_torch_tensor(res, "cuda:2", not paddle_tensor.stop_gradient) + + res = paddle2torch(paddle_tensor, target_device="cuda:1", no_gradient=True) + self.check_torch_tensor(res, "cuda:1", False) + + res = paddle2torch(paddle_tensor, target_device="cuda:1", no_gradient=False) + self.check_torch_tensor(res, "cuda:1", True) + + def test_list_transfer(self): + """ + 测试张量列表的转换 + """ + + paddle_list = [paddle.rand((6, 4, 2)).cuda(1) for i in range(10)] + res = paddle2torch(paddle_list) + assert isinstance(res, list) + for t in res: + self.check_torch_tensor(t, "cuda:1", False) + + res = paddle2torch(paddle_list, target_device="cpu", no_gradient=False) + assert isinstance(res, list) + for t in res: + self.check_torch_tensor(t, "cpu", True) + + def test_tensor_tuple_transfer(self): + """ + 测试张量元组的转换 + """ + + paddle_list = [paddle.rand((6, 4, 2)).cuda(1) for i in range(10)] + paddle_tuple = tuple(paddle_list) + res = paddle2torch(paddle_tuple) + assert isinstance(res, tuple) + for t in res: + self.check_torch_tensor(t, "cuda:1", False) + + def test_dict_transfer(self): + """ + 测试包含复杂结构的字典的转换 + """ + + paddle_dict = { + "tensor": paddle.rand((3, 4)).cuda(0), + "list": [paddle.rand((6, 4, 2)).cuda(0) for i in range(10)], + "dict":{ + "list": [paddle.rand((6, 4, 2)).cuda(0) for i in range(10)], + "tensor": paddle.rand((3, 4)).cuda(0) + }, + "int": 2, + "string": "test string" + } + res = paddle2torch(paddle_dict) + assert isinstance(res, dict) + self.check_torch_tensor(res["tensor"], "cuda:0", False) + assert isinstance(res["list"], list) + for t in res["list"]: + self.check_torch_tensor(t, "cuda:0", False) + assert isinstance(res["int"], int) + assert isinstance(res["string"], str) + assert isinstance(res["dict"], dict) + assert isinstance(res["dict"]["list"], list) + for t in res["dict"]["list"]: + self.check_torch_tensor(t, "cuda:0", False) + self.check_torch_tensor(res["dict"]["tensor"], "cuda:0", False) + + +############################################################################ +# +# 测试torch到paddle的转换 +# +############################################################################ + +@pytest.mark.torchpaddle +class TestTorch2Paddle: + + def check_paddle_tensor(self, tensor, device, stop_gradient): + """ + 检查得到的paddle张量设备和梯度情况的工具函数 + """ + + assert isinstance(tensor, paddle.Tensor) + if device == "cpu": + assert tensor.place.is_cpu_place() + elif device.startswith("gpu"): + paddle_device = paddle.device._convert_to_place(device) + assert tensor.place.is_gpu_place() + if hasattr(tensor.place, "gpu_device_id"): + # paddle中，有两种Place + # paddle.fluid.core.Place是创建Tensor时使用的类型 + # 有函数gpu_device_id获取设备 + assert tensor.place.gpu_device_id() == paddle_device.get_device_id() + else: + # 通过_convert_to_place得到的是paddle.CUDAPlace + # 通过get_device_id获取设备 + assert tensor.place.get_device_id() == paddle_device.get_device_id() + else: + raise NotImplementedError + assert tensor.stop_gradient == stop_gradient + + def test_gradient(self): + """ + 测试转换后梯度的反向传播 + """ + + x = torch.tensor([1.0, 2.0, 3.0, 4.0, 5.0], requires_grad=True) + y = torch2paddle(x) + z = 3 * (y ** 2) + z.sum().backward() + assert y.grad.tolist() == [6, 12, 18, 24, 30] + + def test_tensor_transfer(self): + """ + 测试单个张量的转换 + """ + + torch_tensor = torch.rand((3, 4, 5)) + res = torch2paddle(torch_tensor) + self.check_paddle_tensor(res, "cpu", True) + + res = torch2paddle(torch_tensor, target_device="gpu:2", no_gradient=None) + self.check_paddle_tensor(res, "gpu:2", True) + + res = torch2paddle(torch_tensor, target_device="gpu:2", no_gradient=True) + self.check_paddle_tensor(res, "gpu:2", True) + + res = torch2paddle(torch_tensor, target_device="gpu:2", no_gradient=False) + self.check_paddle_tensor(res, "gpu:2", False) + + def test_tensor_list_transfer(self): + """ + 测试张量列表的转换 + """ + + torch_list = [torch.rand(6, 4, 2) for i in range(10)] + res = torch2paddle(torch_list) + assert isinstance(res, list) + for t in res: + self.check_paddle_tensor(t, "cpu", True) + + res = torch2paddle(torch_list, target_device="gpu:1", no_gradient=False) + assert isinstance(res, list) + for t in res: + self.check_paddle_tensor(t, "gpu:1", False) + + def test_tensor_tuple_transfer(self): + """ + 测试张量元组的转换 + """ + + torch_list = [torch.rand(6, 4, 2) for i in range(10)] + torch_tuple = tuple(torch_list) + res = torch2paddle(torch_tuple, target_device="cpu") + assert isinstance(res, tuple) + for t in res: + self.check_paddle_tensor(t, "cpu", True) + + def test_dict_transfer(self): + """ + 测试复杂的字典结构的转换 + """ + + torch_dict = { + "tensor": torch.rand((3, 4)), + "list": [torch.rand(6, 4, 2) for i in range(10)], + "dict":{ + "list": [torch.rand(6, 4, 2) for i in range(10)], + "tensor": torch.rand((3, 4)) + }, + "int": 2, + "string": "test string" + } + res = torch2paddle(torch_dict) + assert isinstance(res, dict) + self.check_paddle_tensor(res["tensor"], "cpu", True) + assert isinstance(res["list"], list) + for t in res["list"]: + self.check_paddle_tensor(t, "cpu", True) + assert isinstance(res["int"], int) + assert isinstance(res["string"], str) + assert isinstance(res["dict"], dict) + assert isinstance(res["dict"]["list"], list) + for t in res["dict"]["list"]: + self.check_paddle_tensor(t, "cpu", True) + self.check_paddle_tensor(res["dict"]["tensor"], "cpu", True) + + +############################################################################ +# +# 测试jittor到torch的转换 +# +############################################################################ + +class TestJittor2Torch: + + def check_torch_tensor(self, tensor, device, requires_grad): + """ + 检查得到的torch张量的工具函数 + """ + + assert isinstance(tensor, torch.Tensor) + if device == "cpu": + assert not tensor.is_cuda + else: + assert tensor.device == torch.device(device) + assert tensor.requires_grad == requires_grad + + def test_var_transfer(self): + """ + 测试单个Jittor Var的转换 + """ + + jittor_var = jittor.rand((3, 4, 5)) + res = jittor2torch(jittor_var) + self.check_torch_tensor(res, "cpu", True) + + res = jittor2torch(jittor_var, target_device="cuda:2", no_gradient=None) + self.check_torch_tensor(res, "cuda:2", True) + + res = jittor2torch(jittor_var, target_device="cuda:2", no_gradient=True) + self.check_torch_tensor(res, "cuda:2", False) + + res = jittor2torch(jittor_var, target_device="cuda:2", no_gradient=False) + self.check_torch_tensor(res, "cuda:2", True) + + def test_var_list_transfer(self): + """ + 测试Jittor列表的转换 + """ + + jittor_list = [jittor.rand((6, 4, 2)) for i in range(10)] + res = jittor2torch(jittor_list) + assert isinstance(res, list) + for t in res: + self.check_torch_tensor(t, "cpu", True) + + res = jittor2torch(jittor_list, target_device="cuda:1", no_gradient=False) + assert isinstance(res, list) + for t in res: + self.check_torch_tensor(t, "cuda:1", True) + + def test_var_tuple_transfer(self): + """ + 测试Jittor变量元组的转换 + """ + + jittor_list = [jittor.rand((6, 4, 2)) for i in range(10)] + jittor_tuple = tuple(jittor_list) + res = jittor2torch(jittor_tuple, target_device="cpu") + assert isinstance(res, tuple) + for t in res: + self.check_torch_tensor(t, "cpu", True) + + def test_dict_transfer(self): + """ + 测试字典结构的转换 + """ + + jittor_dict = { + "tensor": jittor.rand((3, 4)), + "list": [jittor.rand(6, 4, 2) for i in range(10)], + "dict":{ + "list": [jittor.rand(6, 4, 2) for i in range(10)], + "tensor": jittor.rand((3, 4)) + }, + "int": 2, + "string": "test string" + } + res = jittor2torch(jittor_dict) + assert isinstance(res, dict) + self.check_torch_tensor(res["tensor"], "cpu", True) + assert isinstance(res["list"], list) + for t in res["list"]: + self.check_torch_tensor(t, "cpu", True) + assert isinstance(res["int"], int) + assert isinstance(res["string"], str) + assert isinstance(res["dict"], dict) + assert isinstance(res["dict"]["list"], list) + for t in res["dict"]["list"]: + self.check_torch_tensor(t, "cpu", True) + self.check_torch_tensor(res["dict"]["tensor"], "cpu", True) + + +############################################################################ +# +# 测试torch到jittor的转换 +# +############################################################################ + +class TestTorch2Jittor: + + def check_jittor_var(self, var, requires_grad): + """ + 检查得到的Jittor Var梯度情况的工具函数 + """ + + assert isinstance(var, jittor.Var) + assert var.requires_grad == requires_grad + + def test_gradient(self): + """ + 测试反向传播的梯度 + """ + + x = torch.tensor([1.0, 2.0, 3.0, 4.0, 5.0], requires_grad=True) + y = torch2jittor(x) + z = 3 * (y ** 2) + grad = jittor.grad(z, y) + assert grad.tolist() == [6.0, 12.0, 18.0, 24.0, 30.0] + + def test_tensor_transfer(self): + """ + 测试单个张量转换为Jittor + """ + + torch_tensor = torch.rand((3, 4, 5)) + res = torch2jittor(torch_tensor) + self.check_jittor_var(res, False) + + res = torch2jittor(torch_tensor, no_gradient=None) + self.check_jittor_var(res, False) + + res = torch2jittor(torch_tensor, no_gradient=True) + self.check_jittor_var(res, False) + + res = torch2jittor(torch_tensor, no_gradient=False) + self.check_jittor_var(res, True) + + def test_tensor_list_transfer(self): + """ + 测试张量列表的转换 + """ + + torch_list = [torch.rand((6, 4, 2)) for i in range(10)] + res = torch2jittor(torch_list) + assert isinstance(res, list) + for t in res: + self.check_jittor_var(t, False) + + res = torch2jittor(torch_list, no_gradient=False) + assert isinstance(res, list) + for t in res: + self.check_jittor_var(t, True) + + def test_tensor_tuple_transfer(self): + """ + 测试张量元组的转换 + """ + + torch_list = [torch.rand((6, 4, 2)) for i in range(10)] + torch_tuple = tuple(torch_list) + res = torch2jittor(torch_tuple) + assert isinstance(res, tuple) + for t in res: + self.check_jittor_var(t, False) + + def test_dict_transfer(self): + """ + 测试字典结构的转换 + """ + + torch_dict = { + "tensor": torch.rand((3, 4)), + "list": [torch.rand(6, 4, 2) for i in range(10)], + "dict":{ + "list": [torch.rand(6, 4, 2) for i in range(10)], + "tensor": torch.rand((3, 4)) + }, + "int": 2, + "string": "test string" + } + res = torch2jittor(torch_dict) + assert isinstance(res, dict) + self.check_jittor_var(res["tensor"], False) + assert isinstance(res["list"], list) + for t in res["list"]: + self.check_jittor_var(t, False) + assert isinstance(res["int"], int) + assert isinstance(res["string"], str) + assert isinstance(res["dict"], dict) + assert isinstance(res["dict"]["list"], list) + for t in res["dict"]["list"]: + self.check_jittor_var(t, False) + self.check_jittor_var(res["dict"]["tensor"], False) From 8c5ac2776cdf8fbf24812287300b675bec720f28 Mon Sep 17 00:00:00 2001 From: yh_cc Date: Tue, 10 May 2022 21:09:49 +0800 Subject: [PATCH 15/15] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E9=83=A8=E5=88=86?= =?UTF-8?q?=E6=96=87=E6=A1=A3;=20Trainer=20;Evaluator=E6=8A=A5=E9=94=99?= =?UTF-8?q?=E4=BC=9A=E5=B0=9D=E8=AF=95=E6=89=93=E5=8D=B0indices?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../controllers/loops/evaluate_batch_loop.py | 2 +- .../controllers/loops/train_batch_loop.py | 2 +- fastNLP/core/controllers/trainer.py | 2 +- fastNLP/core/log/print.py | 6 ++-- fastNLP/core/metrics/metric.py | 2 +- .../samplers/reproducible_batch_sampler.py | 29 ++++++++++++------- fastNLP/core/samplers/reproducible_sampler.py | 28 +++++++++++------- 7 files changed, 44 insertions(+), 27 deletions(-) diff --git a/fastNLP/core/controllers/loops/evaluate_batch_loop.py b/fastNLP/core/controllers/loops/evaluate_batch_loop.py index 0bf66fda..80c234cd 100644 --- a/fastNLP/core/controllers/loops/evaluate_batch_loop.py +++ b/fastNLP/core/controllers/loops/evaluate_batch_loop.py @@ -34,7 +34,7 @@ class EvaluateBatchLoop(Loop): except BaseException as e: if callable(getattr(dataloader, 'get_batch_indices', None)): indices = dataloader.get_batch_indices() - logger.debug(f"The following exception happens when running on samples: {indices}") + logger.error(f"Exception happens when evaluating on samples: {indices}") raise e self.batch_step_fn(evaluator, batch) diff --git a/fastNLP/core/controllers/loops/train_batch_loop.py b/fastNLP/core/controllers/loops/train_batch_loop.py index ef05e0c4..989fb2ae 100644 --- a/fastNLP/core/controllers/loops/train_batch_loop.py +++ b/fastNLP/core/controllers/loops/train_batch_loop.py @@ -32,7 +32,7 @@ class TrainBatchLoop(Loop): break except BaseException as e: if indices and not isinstance(e, EarlyStopException): - logger.debug(f"The following exception happens when running on samples: {indices}") + logger.error(f"Exception happens when running on samples: {indices}") raise e trainer.on_train_batch_begin(batch, indices) diff --git a/fastNLP/core/controllers/trainer.py b/fastNLP/core/controllers/trainer.py index 2116674f..9d49641b 100644 --- a/fastNLP/core/controllers/trainer.py +++ b/fastNLP/core/controllers/trainer.py @@ -514,7 +514,7 @@ class Trainer(TrainerEventTrigger): else: raise FileNotFoundError("You are using `resume_from`, but we can not find your specific file.") - if self.evaluator is not None and num_eval_sanity_batch > 0: + if self.evaluator is not None and num_eval_sanity_batch != 0: logger.info(f"Running evaluator sanity check for {num_eval_sanity_batch} batches.") self.on_sanity_check_begin() sanity_check_res = self.evaluator.run(num_eval_batch_per_dl=num_eval_sanity_batch) diff --git a/fastNLP/core/log/print.py b/fastNLP/core/log/print.py index b3d328ed..f40d763e 100644 --- a/fastNLP/core/log/print.py +++ b/fastNLP/core/log/print.py @@ -1,7 +1,7 @@ __all__ = [ 'print' ] - +from logging import INFO from .logger import logger @@ -22,4 +22,6 @@ def print(*args, sep=' ', end='\n', file=None, flush=False): :return: """ line = sep.join(map(str, args)) - logger.info(line) \ No newline at end of file + if logger.isEnabledFor(INFO): + kwargs = logger._add_rank_info({}) + logger._log(INFO, line, args, **kwargs) diff --git a/fastNLP/core/metrics/metric.py b/fastNLP/core/metrics/metric.py index 6a32ef60..87505be1 100644 --- a/fastNLP/core/metrics/metric.py +++ b/fastNLP/core/metrics/metric.py @@ -84,7 +84,7 @@ class Metric: def _sync_get_metric(self, get_metric): @functools.wraps(get_metric) def _wrap_get_metric(*args, **kwargs): - assert self._updated, f"You have to call `{self.__class__.__name__}` update() function before calling " \ + assert self._updated, f"You have to call `{self.__class__.__name__}'s update() function before calling " \ f"get_metric()." with self.sync(recover=True, aggregate=self.aggregate_when_get_metric): results = get_metric(*args, **kwargs) diff --git a/fastNLP/core/samplers/reproducible_batch_sampler.py b/fastNLP/core/samplers/reproducible_batch_sampler.py index 12fe47e7..143a5438 100644 --- a/fastNLP/core/samplers/reproducible_batch_sampler.py +++ b/fastNLP/core/samplers/reproducible_batch_sampler.py @@ -366,17 +366,22 @@ class BucketedBatchSampler(ReproducibleBatchSampler): def __init__(self, dataset, length: Union[List[int], str], batch_size:int = 32, num_batch_per_bucket:int = 10, shuffle: bool = True, drop_last: bool = False, seed: int = 0, **kwargs): """ - 首先按照 sample 的长度排序，然后按照 batch_size*num_batch_per_bucket 为一个桶的大小，sample 只会在这个桶内进行组合，这样 - 每个 batch 中的 padding 数量会比较少 （因为桶内的数据的长度都接近）。 + 首先按照 ``sample`` 的长度排序，然后按照 batch_size*num_batch_per_bucket 为一个桶的大小，``sample`` 只会在这个桶内进行组 + 合，这样每个 ``batch`` 中的 ``padding`` 数量会比较少 （因为桶内的数据的长度都接近）。 :param dataset: 实现了 __len__ 方法的数据容器。 - :param length: 如果为 List，应当与 dataset 有一样的长度，表示 dataset 中每个元素的数量；仅当传入的 dataset 为 fastNLP 的 - DataSet 时支持传入 str，会将该str理解为 dataset 的 field 名称，若 field 中的元素为 int，则认为该值是 sample 的长度。 - 如果否则使用 len() 函数得到每个 sample 中这个 field 的长度。 + :param length: 每条数据的长度。 + + * 为 ``List[int]`` 时 + 应当与 dataset 有一样的长度，表示 dataset 中每个元素的数量； + * 为 ``str`` 时 + 仅当传入的 ``dataset`` 是 :class:`fastNLP.DataSet` 时，允许传入 `str` ，该 `str` 将被认为是 ``dataset`` 中的 + ``field`` 。若 field 中的元素为 ``int``，则认为该值是 sample 的长度；若不为 ``int`` ，则尝试使用 ``len`` 方法 + 获取该 ``field`` 中每个元素的长度。 :param batch_size: 每个 batch 的大小 - :param num_batch_per_bucket: 多少个 batch 组成一个桶，数据只会在一个桶内进行 shuffle 。 - :param shuffle: 如果为 True，将不进行 shuffle，实际上数据会以从长到短的方式输出。 - :param drop_last: 如果最后一个 batch 的 sample 数量无法凑齐 batch_size 这么多，是否需要丢掉。 + :param num_batch_per_bucket: 多少个 ``batch`` 组成一个桶，数据只会在一个桶内进行 ``shuffle`` 。 + :param shuffle: 如果为 True，将不进行 ``shuffle``，实际上数据会以从长到短的方式输出。 + :param drop_last: 如果最后一个 `batch` 的 ``sample`` 数量无法凑齐 ``batch_size`` 这么多，是否需要丢掉。 :param seed: 设置的随机数种子 :param kwargs: fastNLP 保留使用 """ @@ -386,10 +391,12 @@ class BucketedBatchSampler(ReproducibleBatchSampler): if not isinstance(length[0], int): length = list(map(len, length)) else: - assert len(length) == len(dataset), "When the dataset is not fastNLP.DataSet, " \ - "the length parameter can only be List[int]" + types = set(map(type, length)) + assert isinstance(length, list) and len(types)==1 and types.pop()==int, \ + "When the dataset is not fastNLP.DataSet, the length parameter can only be List[int]" - assert len(length) == len(dataset), "The length of `data` and `length` should be equal." + assert len(length) == len(dataset), f"The length of `dataset`({len(dataset)}) and " \ + f"`length`({len(length)}) should be equal." self.dataset = dataset self.length = np.array(length, dtype=int) # 按照长到短排列的序号。 diff --git a/fastNLP/core/samplers/reproducible_sampler.py b/fastNLP/core/samplers/reproducible_sampler.py index 0b2b044b..5972ce70 100644 --- a/fastNLP/core/samplers/reproducible_sampler.py +++ b/fastNLP/core/samplers/reproducible_sampler.py @@ -55,6 +55,7 @@ class ReproducibleSampler: class RandomSampler(ReproducibleSampler): def __init__(self, dataset, shuffle: bool = True, seed: int = 0, **kwargs): """ + 随机顺序的 Sampler 。 :param dataset: 实现了 __len__ 方法的数据容器 :param shuffle: 是否在每次 iterate 的时候打乱顺序。 @@ -169,9 +170,8 @@ class RandomSampler(ReproducibleSampler): def set_epoch(self, epoch: int) -> None: self.epoch = epoch - def set_distributed(self, num_replicas, rank, pad=True): + def set_distributed(self, num_replicas:int, rank:int, pad:bool=True): """ - 该方法本质上等同于 ddp 情形下的没有完成的初始化，应当在初始化该 sampler 本身后立即被调用； :param num_replicas: :param rank: @@ -215,7 +215,7 @@ class RandomSampler(ReproducibleSampler): class SequentialSampler(RandomSampler): def __init__(self, dataset, **kwargs): """ - 按照顺序读取 dataset 。在多卡情况下，间隔读取，例如，在两卡情况下，卡0取 [0,2,4,..], 卡1取 [1,3,5...]。 + 按照顺序读取 ``dataset`` 。在多卡情况下，间隔读取，例如，在两卡情况下，卡 0 取 ``[0,2,4,..]``, 卡1取 ``[1,3,5...]`` 。 :param dataset: 实现了 __len__ 方法的数据容器。 :param kwargs: @@ -285,13 +285,20 @@ class SequentialSampler(RandomSampler): class SortedSampler(SequentialSampler): def __init__(self, dataset, length:Union[str, List], **kwargs): """ - 将 dataset 中的数据根据 length 从长到短进行迭代。在多卡情况下，由于padding 最后一个 sample 可能是最长的那个 sample。 + 将 ``dataset`` 中的数据根据 ``length`` 从长到短进行迭代。在多卡情况下，由于 ``padding`` , 最后一个 ``sample`` 可能是最长 + 的那个 ``sample`` 。 :param dataset: 实现了 __len__ 方法的数据容器。 - :param length: 如果为 List，应当与 dataset 有一样的长度，表示 dataset 中每个元素的数量；仅当传入的 dataset 为 fastNLP 的 - DataSet 时支持传入 str，会将该str理解为 dataset 的 field 名称，若 field 中的元素为 int，则认为该值是 sample 的长度。 - :param seed: 设置的随机数种子 - :param kwargs: fastNLP 保留使用 + :param length: 每条数据的长度。 + + * 为 ``List[int]`` 时 + 应当与 dataset 有一样的长度，表示 dataset 中每个元素的数量； + * 为 ``str`` 时 + 仅当传入的 ``dataset`` 是 :class:`fastNLP.DataSet` 时，允许传入 `str` ，该 `str` 将被认为是 ``dataset`` 中的 + ``field`` 。若 field 中的元素为 ``int``，则认为该值是 sample 的长度；若不为 ``int`` ，则尝试使用 ``len`` 方法 + 获取该 ``field`` 中每个元素的长度。 + :param seed: 设置的随机数种子。 + :param kwargs: fastNLP 保留使用。 """ super().__init__(dataset=dataset, **kwargs) if isinstance(dataset, DataSet) and isinstance(length, str): @@ -299,8 +306,9 @@ class SortedSampler(SequentialSampler): if not isinstance(length[0], int): length = list(map(len, length)) else: - assert len(length) == len(dataset), "When the dataset is not fastNLP.DataSet, " \ - "the length parameter can only be List[int]" + types = set(map(type, length)) + assert isinstance(length, list) and len(types)==1 and types.pop()==int, \ + "When the dataset is not fastNLP.DataSet, the length parameter can only be List[int]" assert len(length) == len(dataset), "The length of `data` and `length` should be equal."