From 34698f0f9e9ecb43d9c7b4ec822615fa779cb0e5 Mon Sep 17 00:00:00 2001 From: x54-729 <17307130121@fudan.edu.cn> Date: Mon, 9 May 2022 11:34:15 +0000 Subject: [PATCH] =?UTF-8?q?=E5=88=A0=E9=99=A4=20mix=5Fmodule=20=E5=92=8C?= =?UTF-8?q?=20torch=5Fpaddle=5Fdriver=20=E7=9A=84=E5=86=85=E5=AE=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/core/__init__.py | 1 - fastNLP/core/drivers/__init__.py | 2 - .../drivers/torch_paddle_driver/__init__.py | 5 - .../torch_paddle_driver.py | 193 -------- .../core/drivers/torch_paddle_driver/utils.py | 4 - fastNLP/core/utils/__init__.py | 2 - fastNLP/core/utils/torch_paddle_utils.py | 49 -- fastNLP/modules/__init__.py | 9 - fastNLP/modules/mix_modules/__init__.py | 10 - fastNLP/modules/mix_modules/mix_module.py | 310 ------------- fastNLP/modules/mix_modules/utils.py | 233 ---------- .../drivers/torch_paddle_driver/__init__.py | 0 .../_test_torch_paddle_driver.py | 122 ----- .../torch_paddle_driver/_test_utils.py | 0 tests/core/utils/_test_torch_paddle_utils.py | 204 -------- tests/modules/__init__.py | 0 tests/modules/mix_modules/__init__.py | 0 tests/modules/mix_modules/_test_mix_module.py | 378 --------------- tests/modules/mix_modules/_test_utils.py | 435 ------------------ 19 files changed, 1957 deletions(-) delete mode 100644 fastNLP/core/drivers/torch_paddle_driver/__init__.py delete mode 100644 fastNLP/core/drivers/torch_paddle_driver/torch_paddle_driver.py delete mode 100644 fastNLP/core/drivers/torch_paddle_driver/utils.py delete mode 100644 fastNLP/core/utils/torch_paddle_utils.py delete mode 100644 fastNLP/modules/__init__.py delete mode 100644 fastNLP/modules/mix_modules/__init__.py delete mode 100644 fastNLP/modules/mix_modules/mix_module.py delete mode 100644 fastNLP/modules/mix_modules/utils.py delete mode 100644 tests/core/drivers/torch_paddle_driver/__init__.py delete mode 100644 tests/core/drivers/torch_paddle_driver/_test_torch_paddle_driver.py delete mode 100644 tests/core/drivers/torch_paddle_driver/_test_utils.py delete mode 100644 tests/core/utils/_test_torch_paddle_utils.py delete mode 100644 tests/modules/__init__.py delete mode 100644 tests/modules/mix_modules/__init__.py delete mode 100644 tests/modules/mix_modules/_test_mix_module.py delete mode 100644 tests/modules/mix_modules/_test_utils.py diff --git a/fastNLP/core/__init__.py b/fastNLP/core/__init__.py index 8800be8e..b0f71f52 100644 --- a/fastNLP/core/__init__.py +++ b/fastNLP/core/__init__.py @@ -63,7 +63,6 @@ __all__ = [ "PaddleFleetDriver", "JittorSingleDriver", "JittorMPIDriver", - "TorchPaddleDriver", # log "logger", diff --git a/fastNLP/core/drivers/__init__.py b/fastNLP/core/drivers/__init__.py index a67d886e..f9be3180 100644 --- a/fastNLP/core/drivers/__init__.py +++ b/fastNLP/core/drivers/__init__.py @@ -9,7 +9,6 @@ __all__ = [ "JittorDriver", "JittorSingleDriver", "JittorMPIDriver", - "TorchPaddleDriver", 'torch_seed_everything', 'paddle_seed_everything', 'optimizer_state_to_device' @@ -18,7 +17,6 @@ __all__ = [ from .torch_driver import TorchDriver, TorchSingleDriver, TorchDDPDriver, torch_seed_everything, optimizer_state_to_device from .jittor_driver import JittorDriver, JittorMPIDriver, JittorSingleDriver from .paddle_driver import PaddleDriver, PaddleFleetDriver, PaddleSingleDriver, paddle_seed_everything -from .torch_paddle_driver import TorchPaddleDriver from .driver import Driver diff --git a/fastNLP/core/drivers/torch_paddle_driver/__init__.py b/fastNLP/core/drivers/torch_paddle_driver/__init__.py deleted file mode 100644 index 6deeed73..00000000 --- a/fastNLP/core/drivers/torch_paddle_driver/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -__all__ = [ - "TorchPaddleDriver", -] - -from .torch_paddle_driver import TorchPaddleDriver \ No newline at end of file diff --git a/fastNLP/core/drivers/torch_paddle_driver/torch_paddle_driver.py b/fastNLP/core/drivers/torch_paddle_driver/torch_paddle_driver.py deleted file mode 100644 index 20be8a37..00000000 --- a/fastNLP/core/drivers/torch_paddle_driver/torch_paddle_driver.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Optional, Dict, Union, Callable, Tuple - -from fastNLP.envs.imports import _NEED_IMPORT_PADDLE, _NEED_IMPORT_TORCH -from fastNLP.core.utils.utils import _get_fun_msg - - -if _NEED_IMPORT_PADDLE: - import paddle - from paddle.io import DataLoader as PaddleDataLoader - from paddle.optimizer import Optimizer as PaddleOptimizer - -if _NEED_IMPORT_TORCH: - import torch - from torch.utils.data import DataLoader as TorchDataLoader - from torch.optim import Optimizer as TorchOptimizer - -from fastNLP.core.drivers.driver import Driver -from fastNLP.envs.distributed import rank_zero_call -from fastNLP.core.utils.utils import auto_param_call, apply_to_collection -from fastNLP.core.log.logger import logger -from fastNLP.modules.mix_modules.mix_module import MixModule - - -__all__ = [ - "TorchPaddleDriver", -] - -class TorchPaddleDriver(Driver): - """ - 针对torch和paddle混合模型的driver - 由于是两种不同的框架不方便实现多卡，暂时先实现CPU和GPU单卡的功能 - """ - def __init__(self, model, device: Optional[str] = None, **kwargs): - super(TorchPaddleDriver, self).__init__(model) - - self.model_device = device - self.torch_non_blocking = kwargs.get("torch_non_blocking", None) - self.paddle_blocking = kwargs.get("paddle_blocking", None) - - self._data_device = kwargs.get("_data_device", None) - if isinstance(self._data_device, int): - # 将data_device设置为cuda:x的字符串形式 - if self._data_device < 0: - raise ValueError("Parameter `_data_device` can not be smaller than 0.") - _could_use_device_num = paddle.device.cuda.device_count() - if self._data_device >= _could_use_device_num: - raise ValueError("The gpu device that parameter `device` specifies is not existed.") - self._data_device = f"cuda:{self._data_device}" - elif self._data_device is not None: - raise ValueError("Parameter `device` is wrong type, please check our documentation for the right use.") - - def setup(self): - if self.model_device is not None: - paddle.device.set_device(self.model_device.replace("cuda", "gpu")) - self.model.to(self.model_device) - - @staticmethod - def check_dataloader_legality(dataloader, dataloader_name, is_train: bool = False): - if is_train: - if not isinstance(dataloader, (TorchDataLoader, PaddleDataLoader)): - raise ValueError(f"Parameter `{dataloader_name}` should be 'torch.util.data.DataLoader' or `paddle.io.dataloader` type, not {type(dataloader)}.") - else: - if not isinstance(dataloader, Dict): - raise ValueError(f"Parameter `{dataloader_name}` should be 'Dict' type, not {type(dataloader)}.") - else: - for each_dataloader in dataloader.values(): - if not isinstance(each_dataloader, (TorchDataLoader, PaddleDataLoader)): - raise ValueError(f"Each dataloader of parameter `{dataloader_name}` should be " - f"'torch.util.data.DataLoader' or `paddle.io.dataloader` " - f"type, not {type(each_dataloader)}.") - - @staticmethod - def _check_optimizer_legality(optimizers): - for each_optimizer in optimizers: - if not isinstance(each_optimizer, (TorchOptimizer, PaddleOptimizer)): - raise ValueError(f"Each optimizers of parameter `optimizers` should be " - f"'torch.optim.Optimizer' or 'paddle.optimizers.Optimizer' type, " - f"not {type(each_optimizer)}.") - - def step(self): - for optimizer in self.optimizers: - optimizer.step() - - def backward(self, loss): - loss.backward() - - def zero_grad(self): - for optimizer in self.optimizers: - if isinstance(optimizer, TorchOptimizer): - optimizer.zero_grad() - elif isinstance(optimizer, PaddleOptimizer): - optimizer.clear_grad() - else: - raise ValueError("Unknown optimizers type.") - - def model_call(self, batch, fn: Callable, signature_fn: Optional[Callable]) -> Dict: - if isinstance(batch, Dict) and not self.wo_auto_param_call: - return auto_param_call(fn, batch, signature_fn=signature_fn) - else: - return fn(batch) - - def get_model_call_fn(self, fn: str) -> Tuple: - if hasattr(self.model, fn): - fn = getattr(self.model, fn) - if not callable(fn): - raise RuntimeError(f"The `{fn}` attribute is not `Callable`.") - logger.debug(f'Use {_get_fun_msg(fn, with_fp=False)}...') - return fn, None - elif fn in {"train_step", "evaluate_step"}: - logger.debug(f'Use {_get_fun_msg(self.model.forward, with_fp=False)}...') - return self.model, self.model.forward - else: - raise RuntimeError(f"There is no `{fn}` method in your {type(self.model)}.") - - def predict_step(self, batch): - if isinstance(batch, Dict): - return auto_param_call(self._predict_step, batch) - else: - return self._predict_step(batch) - - @rank_zero_call - def save_model(self, filepath: str, only_state_dict: bool = True, model_save_fn: Optional[Callable] = None): - r""" - 暂时不提供保存整个模型的方法 - """ - if only_state_dict == False: - logger.warn("TorchPaddleModule only support saving state dicts now.") - if model_save_fn is not None: - model_save_fn(filepath) - else: - model = self.unwrap_model() - self.move_model_to_device(model, "cpu") - self.model.save(filepath) - self.move_model_to_device(model, self.model_device) - - def load_model(self, filepath: str): - """ - 加载模型的加载函数； - - :param filepath: 保存文件的文件位置（需要包括文件名）； - :return: - """ - return self.model.load(filepath) - - def save(self): - ... - - def load(self): - ... - - @staticmethod - def move_model_to_device(model: MixModule, device: str): - if device is not None: - model.to(device) - - def unwrap_model(self): - return self.model - - @staticmethod - def tensor_to_numeric(tensor): - if tensor is None: - return None - - def _translate(_data): - return _data.tolist() - - return apply_to_collection( - data=tensor, - dtype=(paddle.Tensor, torch.Tensor), - function=_translate - ) - - def set_model_mode(self, mode: str): - assert mode in {"train", "eval"} - getattr(self.model, mode)() - - def get_model_device(self): - return self.model_device - - @property - def data_device(self): - if self.model_device is not None: - return self.model_device - else: - return self._data_device - - def set_model_mode(self, mode: str): - assert mode in {"train", "eval"} - getattr(self.model, mode)() - - def set_sampler_epoch(self, dataloader: Union['TorchDataLoader', 'PaddleDataLoader'], cur_epoch_idx): - # 保证 ddp 训练时的 shuffle=True 时的正确性，因为需要保证每一个进程上的 sampler 的shuffle 的随机数种子是一样的； - return dataloader diff --git a/fastNLP/core/drivers/torch_paddle_driver/utils.py b/fastNLP/core/drivers/torch_paddle_driver/utils.py deleted file mode 100644 index 328ac7ec..00000000 --- a/fastNLP/core/drivers/torch_paddle_driver/utils.py +++ /dev/null @@ -1,4 +0,0 @@ -from fastNLP.envs.imports import _NEED_IMPORT_PADDLE - -if _NEED_IMPORT_PADDLE: - pass \ No newline at end of file diff --git a/fastNLP/core/utils/__init__.py b/fastNLP/core/utils/__init__.py index 4de52d16..aca01344 100644 --- a/fastNLP/core/utils/__init__.py +++ b/fastNLP/core/utils/__init__.py @@ -11,7 +11,6 @@ __all__ = [ 'is_in_fnlp_paddle_dist', 'is_in_paddle_launch_dist', 'f_rich_progress', - 'torch_paddle_move_data_to_device', 'torch_move_data_to_device', 'get_fn_arg_names', 'auto_param_call', @@ -32,7 +31,6 @@ from .jittor_utils import is_jittor_dataset, jittor_collate_wraps from .paddle_utils import get_device_from_visible, paddle_to, paddle_move_data_to_device, get_paddle_device_id, get_paddle_gpu_str, is_in_paddle_dist, \ is_in_fnlp_paddle_dist, is_in_paddle_launch_dist from .rich_progress import f_rich_progress -from .torch_paddle_utils import torch_paddle_move_data_to_device from .torch_utils import torch_move_data_to_device from .utils import * diff --git a/fastNLP/core/utils/torch_paddle_utils.py b/fastNLP/core/utils/torch_paddle_utils.py deleted file mode 100644 index 9201548d..00000000 --- a/fastNLP/core/utils/torch_paddle_utils.py +++ /dev/null @@ -1,49 +0,0 @@ -from typing import Any, Optional - -from fastNLP.envs.imports import _NEED_IMPORT_PADDLE, _NEED_IMPORT_TORCH - -if _NEED_IMPORT_PADDLE: - import paddle - -if _NEED_IMPORT_TORCH: - import torch - -__all__ = [ - "torch_paddle_move_data_to_device", -] - -from .utils import apply_to_collection -from .paddle_utils import paddle_to - - -def torch_paddle_move_data_to_device(batch: Any, device: Optional[str] = None, non_blocking: Optional[bool] = True, - data_device: Optional[str] = None) -> Any: - - r""" - 将数据集合传输到给定设备。只有paddle.Tensor和torch.Tensor对象会被传输到设备中，其余保持不变 - - :param batch: - :param device: - :param non_blocking: - :param data_device: - :return: 相同的集合，但所有包含的张量都驻留在新设备上； - """ - - if device is None: - if data_device is not None: - device = data_device - else: - return batch - - torch_device = device.replace("gpu", "cuda") - paddle_device = device.replace("cuda", "gpu") - - def batch_to(data: Any) -> Any: - if isinstance(data, torch.Tensor): - data = data.to(torch_device, non_blocking=non_blocking) - elif isinstance(data, paddle.Tensor): - data = paddle_to(data, paddle_device) - - return data - - return apply_to_collection(batch, dtype=(paddle.Tensor, torch.Tensor), function=batch_to) \ No newline at end of file diff --git a/fastNLP/modules/__init__.py b/fastNLP/modules/__init__.py deleted file mode 100644 index a2da19c1..00000000 --- a/fastNLP/modules/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -__all__ = [ - "MixModule", - "torch2paddle", - "paddle2torch", - "torch2jittor", - "jittor2torch", -] - -from .mix_modules import MixModule, torch2paddle, paddle2torch, torch2jittor, jittor2torch \ No newline at end of file diff --git a/fastNLP/modules/mix_modules/__init__.py b/fastNLP/modules/mix_modules/__init__.py deleted file mode 100644 index 1e3b085d..00000000 --- a/fastNLP/modules/mix_modules/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -__all__ = [ - "MixModule", - "torch2paddle", - "paddle2torch", - "torch2jittor", - "jittor2torch", -] - -from .mix_module import MixModule -from .utils import * \ No newline at end of file diff --git a/fastNLP/modules/mix_modules/mix_module.py b/fastNLP/modules/mix_modules/mix_module.py deleted file mode 100644 index 40abcf51..00000000 --- a/fastNLP/modules/mix_modules/mix_module.py +++ /dev/null @@ -1,310 +0,0 @@ -import os -import io -import pickle -from typing import Dict -from collections import OrderedDict - -import numpy as np - -from fastNLP.envs.imports import _NEED_IMPORT_JITTOR, _NEED_IMPORT_PADDLE, _NEED_IMPORT_TORCH -from fastNLP.core.utils.paddle_utils import paddle_to - -if _NEED_IMPORT_PADDLE: - import paddle - from paddle.nn import Layer as PaddleLayer - -if _NEED_IMPORT_TORCH: - import torch - from torch.nn import Module as TorchModule, Parameter as TorchParameter - -if _NEED_IMPORT_JITTOR: - import jittor - - -__all__ = [ - "MixModule", -] - -class MixModule: - """ - TODO: 支持不同的混合方式；添加state_dict的支持；如果参数里有List of Tensors该怎么处理； - 是否需要仿照Module那样在初始化的时候给各种模型分类 - 可以同时使用Torch和Paddle框架的混合模型 - """ - def __init__(self, *args, **kwargs): - pass - - def __call__(self, *args, **kwargs): - return self.forward(*args, **kwargs) - - def named_parameters(self, prefix='', recurse: bool=True, backend=None): - """ - 返回模型的名字和参数 - - :param prefix: 输出时在参数名前加上的前缀 - :param recurse: 是否递归地输出参数 - :param backend: `backend`=`None`时，将所有模型和张量的参数返回； - `backend`=`torch`时，返回`torch`的参数； - `backend`=`paddle`时，返回`paddle`的参数。 - """ - if backend is None: - generator = self.attributes(TorchModule, TorchParameter, PaddleLayer) - elif backend == "torch": - generator = self.attributes(TorchModule, TorchParameter) - elif backend == "paddle": - generator = self.attributes(PaddleLayer) - else: - raise ValueError("Unknown backend parameter.") - - for name, value in generator: - name = prefix + ('.' if prefix else '') + name - if isinstance(value, TorchParameter): - # 非Module/Layer类型，直接输出名字和值 - yield name, value - elif recurse: - # 递归地调用named_parameters - for name_r, value_r in value.named_parameters(name, recurse): - yield name_r, value_r - - def parameters(self, recurse: bool = True, backend: str = None): - """ - 返回模型的参数 - - :param recurse: - :param backend: `backend`=`None`时，将所有模型和张量的参数返回； - `backend`=`torch`时，返回`torch`的参数； - `backend`=`paddle`时，返回`paddle`的参数。 - """ - for name, value in self.named_parameters(recurse=recurse, backend=backend): - yield value - - def forward(self, *args, **kwargs): - raise NotImplementedError - - def train_step(self, batch): - raise NotImplementedError - - def test_step(self, batch): - raise NotImplementedError - - def evaluate_step(self, batch): - raise NotImplementedError - - def train(self): - for name, value in self.attributes(TorchModule, PaddleLayer): - value.train() - - def eval(self): - for name, value in self.attributes(TorchModule, PaddleLayer): - value.eval() - - def to(self, device): - """ - :param device: 设备名 - """ - # 有jittor的话 warning - if device == "cpu": - paddle_device = device - elif device.startswith("cuda"): - paddle_device = device.replace("cuda", "gpu") - elif device.startswith("gpu"): - paddle_device = device - device = device.replace("gpu", "cuda") - else: - raise ValueError("Device value error") - - for name, value in self.attributes(TorchModule): - # torch的to函数不影响Tensor - vars(self)[name] = value.to(device) - for name, value in self.attributes(TorchParameter): - # Parameter在经过to函数后会变成Tensor类型 - vars(self)[name] = TorchParameter(value.to(device), requires_grad=value.requires_grad) - - for name, value in self.attributes(PaddleLayer): - vars(self)[name] = value.to(paddle_device) - for name, value in self.attributes(paddle.Tensor): - # paddle的to函数会影响到Tensor - vars(self)[name] = paddle_to(value, paddle_device) - - return self - - def state_dict(self, backend: str = None) -> Dict: - """ - 返回模型的state_dict。 - - .. note:: torch的destination参数会在将来删除，因此不提供destination参数 - - :param backend: `backend`=`None`时，将所有模型和张量的state dict返回； - `backend`=`torch`时，返回`torch`的state dict； - `backend`=`paddle`时，返回`paddle`的state dict。 - """ - if backend is None: - generator = self.attributes(TorchModule, TorchParameter, PaddleLayer) - elif backend == "torch": - generator = self.attributes(TorchModule, TorchParameter) - elif backend == "paddle": - generator = self.attributes(PaddleLayer) - else: - raise ValueError(f"Unknown backend {backend}.") - - destination = OrderedDict() - - for name, value in generator: - if value is None: - continue - if isinstance(value, TorchParameter): - destination[name] = value - else: - # 不同框架state_dict函数的参数名和顺序不同 - if isinstance(value, PaddleLayer): - kwargs = { - "structured_name_prefix": name + ".", - } - elif isinstance(value, TorchModule): - kwargs = { - "prefix": name + ".", - } - else: - raise ValueError(f"Unknown item type {type(value)}") - destination.update(value.state_dict(**kwargs)) - - return destination - - def save_state_dict_to_file(self, path: str): - """ - 保存模型的state dict到path - """ - # TODO 设备限制 - filename = os.path.basename(path) - if filename == "": - raise ValueError("Received empty filename.") - dirname = os.path.dirname(path) - if dirname and not os.path.exists(dirname): - os.makedirs(dirname) - protocol = 4 - - saved = {} - paddle_dict = self.state_dict(backend="paddle") - torch_dict = self.state_dict(backend="torch") - # 保存paddle部分 - # 调用paddle保存时的处理函数 - paddle_saved_obj = paddle.framework.io._build_saved_state_dict(paddle_dict) - paddle_saved_obj = paddle.fluid.io._unpack_saved_dict(paddle_saved_obj, protocol) - # 将返回的dict保存 - saved["paddle"] = paddle_saved_obj - - # 保存torch部分 - buffer = io.BytesIO() - torch.save(torch_dict, buffer) - saved["torch"] = buffer.getvalue() - - # 保存 - with open(path, "wb") as f: - pickle.dump(saved, f, protocol) - - def load_state_dict_from_file(self, path: str): - """ - 从 `path` 中加载保存的state dict - """ - state_dict = {} - with open(path, "rb") as f: - loaded = pickle.load(f) - # 加载paddle的数据 - paddle_loaded_obj = loaded["paddle"] - paddle_load_result = paddle.fluid.io._pack_loaded_dict(paddle_loaded_obj) - if "StructuredToParameterName@@" in paddle_load_result: - for key in paddle_load_result["StructuredToParameterName@@"]: - if isinstance(paddle_load_result[key], np.ndarray): - paddle_load_result[key] = paddle.to_tensor(paddle_load_result[key]) - state_dict.update(paddle_load_result) - # 加载torch的数据 - torch_loaded_obj = loaded["torch"] - torch_bytes = io.BytesIO(torch_loaded_obj) - torch_load_result = torch.load(torch_bytes) - state_dict.update(torch_load_result) - - self.load_state_dict(state_dict) - - def load_state_dict(self, state_dict): - """ - 从state dict中加载数据 - """ - missing_keys = [] - unexpected_keys = [] - error_msgs = [] - new_state = {} - - local_state = self.state_dict() - - # 对字典内容按前缀进行归类 - for key, value in state_dict.items(): - splited = key.split(".", 1) - if len(splited) == 1: - # 没有前缀，实际上只有torch.nn.Parameter会进入这种情况 - new_state[key] = value - else: - prefix, name = splited - if prefix not in new_state: - new_state[prefix] = {} - new_state[prefix][name] = value - - for key, param in self.attributes(TorchModule, TorchParameter, PaddleLayer): - if key in new_state: - # 在传入的字典中找到了对应的值 - input_param = new_state[key] - if not isinstance(input_param, dict): - # 且不是字典，即上述没有前缀的情况 - # 按照torch.nn.Module._load_from_state_dict进行赋值 - if not torch.overrides.is_tensor_like(input_param): - error_msgs.append('While copying the parameter named "{}", ' - 'expected torch.Tensor or Tensor-like object from checkpoint but ' - 'received {}' - .format(key, type(input_param))) - continue - - # This is used to avoid copying uninitialized parameters into - # non-lazy modules, since they dont have the hook to do the checks - # in such case, it will error when accessing the .shape attribute. - is_param_lazy = torch.nn.parameter.is_lazy(param) - # Backward compatibility: loading 1-dim tensor from 0.3.* to version 0.4+ - if not is_param_lazy and len(param.shape) == 0 and len(input_param.shape) == 1: - input_param = input_param[0] - - if not is_param_lazy and input_param.shape != param.shape: - # local shape should match the one in checkpoint - error_msgs.append('size mismatch for {}: copying a param with shape {} from checkpoint, ' - 'the shape in current model is {}.' - .format(key, input_param.shape, param.shape)) - continue - try: - with torch.no_grad(): - param.copy_(input_param) - except Exception as ex: - error_msgs.append('While copying the parameter named "{}", ' - 'whose dimensions in the model are {} and ' - 'whose dimensions in the checkpoint are {}, ' - 'an exception occurred : {}.' - .format(key, param.size(), input_param.size(), ex.args)) - else: - # 否则在子模块中 - if isinstance(param, TorchModule): - # torch模块 - # 由于paddle没有提供类似strict的参数，因此也不对torch作要求 - param.load_state_dict(input_param, strict=False) - elif isinstance(param, PaddleLayer): - # paddle模块 - param.load_dict(input_param) - else: - missing_keys.append(key) - - if len(error_msgs) > 0: - raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format( - self.__class__.__name__, "\n\t".join(error_msgs))) - - def attributes(self, *types): - """ - 查找对应类型的成员 - """ - for name, value in vars(self).items(): - if isinstance(value, types): - yield name, value diff --git a/fastNLP/modules/mix_modules/utils.py b/fastNLP/modules/mix_modules/utils.py deleted file mode 100644 index 5d56ffee..00000000 --- a/fastNLP/modules/mix_modules/utils.py +++ /dev/null @@ -1,233 +0,0 @@ -import warnings -import os -from typing import Any, Optional, Union - -import numpy as np - -from fastNLP.core.utils.utils import apply_to_collection -from fastNLP.core.utils.paddle_utils import paddle_to -from fastNLP.envs.imports import _NEED_IMPORT_JITTOR, _NEED_IMPORT_TORCH, _NEED_IMPORT_PADDLE - -if _NEED_IMPORT_PADDLE: - import paddle - -if _NEED_IMPORT_JITTOR: - import jittor - -if _NEED_IMPORT_TORCH: - import torch - -__all__ = [ - "paddle2torch", - "torch2paddle", - "jittor2torch", - "torch2jittor", -] - -def _paddle2torch(paddle_tensor: 'paddle.Tensor', target_device: Optional[Union[str, int]] = None, no_gradient: bool = None) -> 'torch.Tensor': - """ - 将paddle tensor转换为torch tensor，并且能够保留梯度进行反向传播 - :param paddle_tensor: 要转换的paddle张量 - :param target_device: 是否将转换后的张量迁移到特定设备上，输入为`None`时，和输入的张量相同。 - :param no_gradient: 是否保留原张量的梯度。为`None`时，新的张量与输入张量保持一致； - 为`True`时，全部不保留梯度；为`False`时，全部保留梯度。 - :return: 转换后的torch张量 - """ - no_gradient = paddle_tensor.stop_gradient if no_gradient is None else no_gradient - paddle_numpy = paddle_tensor.numpy() - if not np.issubdtype(paddle_numpy.dtype, np.inexact): - no_gradient = True - - if target_device is None: - if paddle_tensor.place.is_gpu_place(): - # paddlepaddle有两种Place，对应不同的device id获取方式 - if hasattr(paddle_tensor.place, "gpu_device_id"): - # paddle.fluid.core_avx.Place - # 在gpu环境下创建张量的话，张量的place是这一类型 - target_device = f"cuda:{paddle_tensor.place.gpu_device_id()}" - else: - # paddle.CUDAPlace - target_device = f"cuda:{paddle_tensor.place.get_device_id()}" - else: - # TODO: 可能需要支持xpu等设备 - target_device = "cpu" - - if not no_gradient: - # 保持梯度，并保持反向传播 - # torch.tensor会保留numpy数组的类型 - torch_tensor = torch.tensor(paddle_numpy, requires_grad=True, device=target_device) - hook = torch_tensor.register_hook( - lambda grad: paddle.autograd.backward(paddle_tensor, paddle.to_tensor(grad.cpu().numpy())) - ) - else: - # 不保留梯度 - torch_tensor = torch.tensor(paddle_numpy, requires_grad=False, device=target_device) - - return torch_tensor - - -def _torch2paddle(torch_tensor: 'torch.Tensor', target_device: str = None, no_gradient: bool = None) -> 'paddle.Tensor': - """ - 将torch tensor转换为paddle tensor，并且能够保留梯度进行反向传播。 - :param torch_tensor: 要转换的torch张量 - :param target_device: 是否将转换后的张量迁移到特定设备上，输入为`None`时，和输入的张量相同。 - :param no_gradient: 是否保留原张量的梯度。为`None`时，新的张量与输入张量保持一致； - 为`True`时，全部不保留梯度；为`False`时，全部保留梯度。 - :return: 转换后的paddle张量 - """ - no_gradient = not torch_tensor.requires_grad if no_gradient is None else no_gradient - if target_device is None: - if torch_tensor.is_cuda: - target_device = f"gpu:{torch_tensor.device.index}" - else: - target_device = "cpu" - - if not no_gradient: - # 保持梯度并保持反向传播 - # paddle的stop_gradient和torch的requires_grad表现是相反的 - paddle_tensor = paddle.to_tensor(torch_tensor.detach().numpy(), stop_gradient=False) - hook = paddle_tensor.register_hook( - lambda grad: torch.autograd.backward(torch_tensor, torch.tensor(grad.numpy())) - ) - else: - paddle_tensor = paddle.to_tensor(torch_tensor.detach().numpy(), stop_gradient=True) - - paddle_tensor = paddle_to(paddle_tensor, target_device) - - return paddle_tensor - - -def _jittor2torch(jittor_var: 'jittor.Var', target_device: Optional[Union[str, int]] = None, no_gradient: bool = None) -> 'torch.Tensor': - """ - 将jittor Var转换为torch tensor，并且能够保留梯度进行反向传播 - :param jittor_var: 要转换的jittor变量 - :param target_device: 是否将转换后的张量迁移到特定设备上，输入为`None`时，根据jittor.flags.use_cuda决定。 - :param no_gradient: 是否保留原张量的梯度。为`None`时，新的张量与输入张量保持一致； - 为`True`时，全部不保留梯度；为`False`时，全部保留梯度。 - :return: 转换后的torch张量 - """ - # TODO: warning：无法保留梯度 - # jittor的grad可以通过callback进行传递 - # 如果outputs有_grad键，可以实现求导 - no_gradient = not jittor_var.requires_grad if no_gradient is None else no_gradient - if no_gradient == False: - warnings.warn("The result tensor will not keep gradients due to differences between jittor and pytorch.") - jittor_numpy = jittor_var.numpy() - if not np.issubdtype(jittor_numpy.dtype, np.inexact): - no_gradient = True - - if target_device is None: - # jittor的设备分配是自动的 - # 根据use_cuda判断 - if jittor.flags.use_cuda: - target_device = "cuda:0" - else: - target_device = "cpu" - - torch_tensor = torch.tensor(jittor_numpy, requires_grad=not no_gradient, device=target_device) - - return torch_tensor - - -def _torch2jittor(torch_tensor: 'torch.Tensor', no_gradient: bool = None) -> 'jittor.Var': - """ - 将torch tensor转换为jittor Var，并且能够保留梯度进行反向传播 - :param torch_tensor: 要转换的torch张量 - :param no_gradient: 是否保留原张量的梯度。为`None`时，新的张量与输入张量保持一致； - 为`True`时，全部不保留梯度；为`False`时，全部保留梯度。 - :return: 转换后的jittor变量 - """ - no_gradient = not torch_tensor.requires_grad if no_gradient is None else no_gradient - - if not no_gradient: - # 保持梯度并保持反向传播 - jittor_var = jittor.Var(torch_tensor.detach().numpy()) - jittor_var.requires_grad = True - hook = jittor_var.register_hook( - lambda grad: torch.autograd.backward(torch_tensor, torch.tensor(grad.numpy())) - ) - else: - jittor_var = jittor.Var(torch_tensor.detach().numpy()) - jittor_var.requires_grad = False - - return jittor_var - - -def torch2paddle(torch_in: Any, target_device: str = None, no_gradient: bool = None) -> Any: - """ - 递归地将输入中包含的torch张量转换为paddle张量 - - :param torch_in: 要转换的包含torch.Tensor类型的变量 - :param target_device: 是否将转换后的张量迁移到特定设备上， - 输入为`None`时，和输入的张量相同， - :param no_gradient: 是否保留原张量的梯度。为`None`时，新的张量与输入张量保持一致； - 为`True`时，全部不保留梯度；为`False`时，全部保留梯度。 - :return: 将所有torch.Tensor转换为paddle.Tensor的张量 - """ - - return apply_to_collection( - torch_in, - dtype=torch.Tensor, - function=_torch2paddle, - target_device=target_device, - no_gradient=no_gradient, - ) - - -def paddle2torch(paddle_in: Any, target_device: str = None, no_gradient: bool = None) -> Any: - """ - 递归地将输入中包含的paddle张量转换为torch张量 - - :param torch_in: 要转换的包含paddle.Tensor类型的变量 - :param target_device: 是否将转换后的张量迁移到特定设备上， - 输入为`None`时，和输入的张量相同， - :param no_gradient: 是否保留原张量的梯度。为`None`时，新的张量与输入张量保持一致； - 为`True`时，全部不保留梯度；为`False`时，全部保留梯度。 - :return: 将所有paddle.Tensor转换为torch.Tensor后的变量 - """ - - return apply_to_collection( - paddle_in, - dtype=paddle.Tensor, - function=_paddle2torch, - target_device=target_device, - no_gradient=no_gradient, - ) - - -def jittor2torch(jittor_in: Any, target_device: str = None, no_gradient: bool = None) -> Any: - """ - 递归地将输入中包含的jittor变量转换为torch张量 - - :param jittor_in: 要转换的jittor变量 - :param target_device: 是否将转换后的张量迁移到特定设备上，输入为`None`时，默认为cuda:0。 - :param no_gradient: 是否保留原张量的梯度。为`None`时，新的张量与输入张量保持一致； - 为`True`时，全部不保留梯度；为`False`时，全部保留梯度。 - :return: 转换后的torch张量 - """ - - return apply_to_collection( - jittor_in, - dtype=jittor.Var, - function=_jittor2torch, - target_device=target_device, - no_gradient=no_gradient, - ) - - -def torch2jittor(torch_in: Any, no_gradient: bool = None) -> Any: - """ - 递归地将输入中包含的torch张量转换为jittor变量 - - :param torch_tensor: 要转换的torch张量 - :param no_gradient: 是否保留原张量的梯度。为`None`时，新的张量与输入张量保持一致； - 为`True`时，全部不保留梯度；为`False`时，全部保留梯度。 - :return: 转换后的jittor变量 - """ - - return apply_to_collection( - torch_in, - dtype=torch.Tensor, - function=_torch2jittor, - no_gradient=no_gradient, - ) \ No newline at end of file diff --git a/tests/core/drivers/torch_paddle_driver/__init__.py b/tests/core/drivers/torch_paddle_driver/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/core/drivers/torch_paddle_driver/_test_torch_paddle_driver.py b/tests/core/drivers/torch_paddle_driver/_test_torch_paddle_driver.py deleted file mode 100644 index 76b19ba4..00000000 --- a/tests/core/drivers/torch_paddle_driver/_test_torch_paddle_driver.py +++ /dev/null @@ -1,122 +0,0 @@ -import pytest - -from fastNLP.modules.mix_modules.mix_module import MixModule -from fastNLP.core.drivers.torch_paddle_driver.torch_paddle_driver import TorchPaddleDriver -from fastNLP.modules.mix_modules.utils import paddle2torch, torch2paddle - -import torch -import paddle -from paddle.io import Dataset, DataLoader -import numpy as np - -############################################################################ -# -# 测试在MNIST数据集上的表现 -# -############################################################################ - -class MNISTDataset(Dataset): - def __init__(self, dataset): - - self.dataset = [ - ( - np.array(img).astype('float32').reshape(-1), - label - ) for img, label in dataset - ] - - def __getitem__(self, idx): - return self.dataset[idx] - - def __len__(self): - return len(self.dataset) - -class MixMNISTModel(MixModule): - def __init__(self): - super(MixMNISTModel, self).__init__() - - self.fc1 = paddle.nn.Linear(784, 64) - self.fc2 = paddle.nn.Linear(64, 32) - self.fc3 = torch.nn.Linear(32, 10) - self.fc4 = torch.nn.Linear(10, 10) - - def forward(self, x): - - paddle_out = self.fc1(x) - paddle_out = self.fc2(paddle_out) - torch_in = paddle2torch(paddle_out) - torch_out = self.fc3(torch_in) - torch_out = self.fc4(torch_out) - - return torch_out - - def train_step(self, x): - return self.forward(x) - - def test_step(self, x): - return self.forward(x) - -@pytest.mark.torchpaddle -class TestMNIST: - - @classmethod - def setup_class(self): - - self.train_dataset = paddle.vision.datasets.MNIST(mode='train') - self.test_dataset = paddle.vision.datasets.MNIST(mode='test') - self.train_dataset = MNISTDataset(self.train_dataset) - - self.lr = 0.0003 - self.epochs = 20 - - self.dataloader = DataLoader(self.train_dataset, batch_size=100, shuffle=True) - - def setup_method(self): - - model = MixMNISTModel() - self.torch_loss_func = torch.nn.CrossEntropyLoss() - - torch_opt = torch.optim.Adam(model.parameters(backend="torch"), self.lr) - paddle_opt = paddle.optimizer.Adam(parameters=model.parameters(backend="paddle"), learning_rate=self.lr) - - self.driver = TorchPaddleDriver(model=model, device="cuda:0") - self.driver.set_optimizers([torch_opt, paddle_opt]) - - def test_case1(self): - - epochs = 20 - - self.driver.setup() - self.driver.zero_grad() - # 开始训练 - current_epoch_idx = 0 - while current_epoch_idx < epochs: - epoch_loss, batch = 0, 0 - self.driver.set_model_mode("train") - self.driver.set_sampler_epoch(self.dataloader, current_epoch_idx) - for batch, (img, label) in enumerate(self.dataloader): - img = paddle.to_tensor(img).cuda() - torch_out = self.driver.train_step(img) - label = torch.from_numpy(label.numpy()).reshape(-1) - loss = self.torch_loss_func(torch_out.cpu(), label) - epoch_loss += loss.item() - - self.driver.backward(loss) - self.driver.step() - self.driver.zero_grad() - - current_epoch_idx += 1 - - # 开始测试 - correct = 0 - for img, label in self.test_dataset: - - img = paddle.to_tensor(np.array(img).astype('float32').reshape(1, -1)) - torch_out = self.driver.test_step(img) - res = torch_out.softmax(-1).argmax().item() - label = label.item() - if res == label: - correct += 1 - - acc = correct / len(self.test_dataset) - assert acc > 0.85 diff --git a/tests/core/drivers/torch_paddle_driver/_test_utils.py b/tests/core/drivers/torch_paddle_driver/_test_utils.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/core/utils/_test_torch_paddle_utils.py b/tests/core/utils/_test_torch_paddle_utils.py deleted file mode 100644 index e10b1d11..00000000 --- a/tests/core/utils/_test_torch_paddle_utils.py +++ /dev/null @@ -1,204 +0,0 @@ -import paddle -import pytest -import torch - -from fastNLP.core.utils.torch_paddle_utils import torch_paddle_move_data_to_device - -############################################################################ -# -# 测试将参数中包含的所有torch和paddle张量迁移到指定设备 -# -############################################################################ - -@pytest.mark.torchpaddle -class TestTorchPaddleMoveDataToDevice: - - def check_gpu(self, tensor, idx): - """ - 检查张量是否在指定显卡上的工具函数 - """ - - if isinstance(tensor, paddle.Tensor): - assert tensor.place.is_gpu_place() - assert tensor.place.gpu_device_id() == idx - elif isinstance(tensor, torch.Tensor): - assert tensor.is_cuda - assert tensor.device.index == idx - - def check_cpu(self, tensor): - if isinstance(tensor, paddle.Tensor): - assert tensor.place.is_cpu_place() - elif isinstance(tensor, torch.Tensor): - assert not tensor.is_cuda - - def test_tensor_transfer(self): - """ - 测试迁移单个张量 - """ - - paddle_tensor = paddle.rand((3, 4, 5)).cpu() - res = torch_paddle_move_data_to_device(paddle_tensor, device=None, data_device=None) - self.check_cpu(res) - - res = torch_paddle_move_data_to_device(paddle_tensor, device="gpu:0", data_device=None) - self.check_gpu(res, 0) - - res = torch_paddle_move_data_to_device(paddle_tensor, device="gpu:1", data_device=None) - self.check_gpu(res, 1) - - res = torch_paddle_move_data_to_device(paddle_tensor, device="cuda:0", data_device="cpu") - self.check_gpu(res, 0) - - res = torch_paddle_move_data_to_device(paddle_tensor, device=None, data_device="gpu:0") - self.check_gpu(res, 0) - - res = torch_paddle_move_data_to_device(paddle_tensor, device=None, data_device="cuda:1") - self.check_gpu(res, 1) - - torch_tensor = torch.rand(3, 4, 5) - res = torch_paddle_move_data_to_device(torch_tensor, device=None, data_device=None) - self.check_cpu(res) - - res = torch_paddle_move_data_to_device(torch_tensor, device="gpu:0", data_device=None) - self.check_gpu(res, 0) - - res = torch_paddle_move_data_to_device(torch_tensor, device="gpu:1", data_device=None) - self.check_gpu(res, 1) - - res = torch_paddle_move_data_to_device(torch_tensor, device="gpu:0", data_device="cpu") - self.check_gpu(res, 0) - - res = torch_paddle_move_data_to_device(torch_tensor, device=None, data_device="gpu:0") - self.check_gpu(res, 0) - - res = torch_paddle_move_data_to_device(torch_tensor, device=None, data_device="gpu:1") - self.check_gpu(res, 1) - - def test_list_transfer(self): - """ - 测试迁移张量的列表 - """ - - paddle_list = [paddle.rand((6, 4, 2)) for i in range(5)] + [torch.rand((6, 4, 2)) for i in range(5)] - res = torch_paddle_move_data_to_device(paddle_list, device=None, data_device="gpu:1") - assert isinstance(res, list) - for r in res: - self.check_gpu(r, 1) - - res = torch_paddle_move_data_to_device(paddle_list, device="cpu", data_device="gpu:1") - assert isinstance(res, list) - for r in res: - self.check_cpu(r) - - res = torch_paddle_move_data_to_device(paddle_list, device="gpu:0", data_device=None) - assert isinstance(res, list) - for r in res: - self.check_gpu(r, 0) - - res = torch_paddle_move_data_to_device(paddle_list, device="gpu:1", data_device="cpu") - assert isinstance(res, list) - for r in res: - self.check_gpu(r, 1) - - def test_tensor_tuple_transfer(self): - """ - 测试迁移张量的元组 - """ - - paddle_list = [paddle.rand((6, 4, 2)) for i in range(10)] + [torch.rand((6, 4, 2)) for i in range(5)] - paddle_tuple = tuple(paddle_list) - res = torch_paddle_move_data_to_device(paddle_tuple, device=None, data_device="gpu:1") - assert isinstance(res, tuple) - for r in res: - self.check_gpu(r, 1) - - res = torch_paddle_move_data_to_device(paddle_tuple, device="cpu", data_device="gpu:1") - assert isinstance(res, tuple) - for r in res: - self.check_cpu(r) - - res = torch_paddle_move_data_to_device(paddle_tuple, device="gpu:0", data_device=None) - assert isinstance(res, tuple) - for r in res: - self.check_gpu(r, 0) - - res = torch_paddle_move_data_to_device(paddle_tuple, device="gpu:1", data_device="cpu") - assert isinstance(res, tuple) - for r in res: - self.check_gpu(r, 1) - - def test_dict_transfer(self): - """ - 测试迁移复杂的字典结构 - """ - - paddle_dict = { - "torch_tensor": torch.rand((3, 4)), - "torch_list": [torch.rand((6, 4, 2)) for i in range(10)], - "dict":{ - "list": [paddle.rand((6, 4, 2)) for i in range(5)] + [torch.rand((6, 4, 2)) for i in range(5)], - "torch_tensor": torch.rand((3, 4)), - "paddle_tensor": paddle.rand((3, 4)) - }, - "paddle_tensor": paddle.rand((3, 4)), - "list": [paddle.rand((6, 4, 2)) for i in range(10)] , - "int": 2, - "string": "test string" - } - - res = torch_paddle_move_data_to_device(paddle_dict, device="gpu:0", data_device=None) - assert isinstance(res, dict) - self.check_gpu(res["torch_tensor"], 0) - self.check_gpu(res["paddle_tensor"], 0) - assert isinstance(res["torch_list"], list) - for t in res["torch_list"]: - self.check_gpu(t, 0) - assert isinstance(res["list"], list) - for t in res["list"]: - self.check_gpu(t, 0) - assert isinstance(res["int"], int) - assert isinstance(res["string"], str) - assert isinstance(res["dict"], dict) - assert isinstance(res["dict"]["list"], list) - for t in res["dict"]["list"]: - self.check_gpu(t, 0) - self.check_gpu(res["dict"]["torch_tensor"], 0) - self.check_gpu(res["dict"]["paddle_tensor"], 0) - - res = torch_paddle_move_data_to_device(paddle_dict, device=None, data_device="gpu:1") - assert isinstance(res, dict) - self.check_gpu(res["torch_tensor"], 1) - self.check_gpu(res["paddle_tensor"], 1) - assert isinstance(res["torch_list"], list) - for t in res["torch_list"]: - self.check_gpu(t, 1) - assert isinstance(res["list"], list) - for t in res["list"]: - self.check_gpu(t, 1) - assert isinstance(res["int"], int) - assert isinstance(res["string"], str) - assert isinstance(res["dict"], dict) - assert isinstance(res["dict"]["list"], list) - for t in res["dict"]["list"]: - self.check_gpu(t, 1) - self.check_gpu(res["dict"]["torch_tensor"], 1) - self.check_gpu(res["dict"]["paddle_tensor"], 1) - - res = torch_paddle_move_data_to_device(paddle_dict, device="cpu", data_device="gpu:0") - assert isinstance(res, dict) - self.check_cpu(res["torch_tensor"]) - self.check_cpu(res["paddle_tensor"]) - assert isinstance(res["torch_list"], list) - for t in res["torch_list"]: - self.check_cpu(t) - assert isinstance(res["list"], list) - for t in res["list"]: - self.check_cpu(t) - assert isinstance(res["int"], int) - assert isinstance(res["string"], str) - assert isinstance(res["dict"], dict) - assert isinstance(res["dict"]["list"], list) - for t in res["dict"]["list"]: - self.check_cpu(t) - self.check_cpu(res["dict"]["torch_tensor"]) - self.check_cpu(res["dict"]["paddle_tensor"]) diff --git a/tests/modules/__init__.py b/tests/modules/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/modules/mix_modules/__init__.py b/tests/modules/mix_modules/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/modules/mix_modules/_test_mix_module.py b/tests/modules/mix_modules/_test_mix_module.py deleted file mode 100644 index 87206fd6..00000000 --- a/tests/modules/mix_modules/_test_mix_module.py +++ /dev/null @@ -1,378 +0,0 @@ -import pytest -import os -from itertools import chain - -import torch -import paddle -from paddle.io import Dataset, DataLoader -import numpy as np - -from fastNLP.modules.mix_modules.mix_module import MixModule -from fastNLP.modules.mix_modules.utils import paddle2torch, torch2paddle -from fastNLP.envs.distributed import rank_zero_rm - - -############################################################################ -# -# 测试类的基本功能 -# -############################################################################ - -class MixModuleForTest(MixModule): - def __init__(self): - super(MixModuleForTest, self).__init__() - - self.torch_fc1 = torch.nn.Linear(10, 10) - self.torch_softmax = torch.nn.Softmax(0) - self.torch_conv2d1 = torch.nn.Conv2d(10, 10, 3) - self.torch_tensor = torch.ones(3, 3) - self.torch_param = torch.nn.Parameter(torch.ones(4, 4)) - - self.paddle_fc1 = paddle.nn.Linear(10, 10) - self.paddle_softmax = paddle.nn.Softmax(0) - self.paddle_conv2d1 = paddle.nn.Conv2D(10, 10, 3) - self.paddle_tensor = paddle.ones((4, 4)) - -class TorchModuleForTest(torch.nn.Module): - def __init__(self): - super(TorchModuleForTest, self).__init__() - - self.torch_fc1 = torch.nn.Linear(10, 10) - self.torch_softmax = torch.nn.Softmax(0) - self.torch_conv2d1 = torch.nn.Conv2d(10, 10, 3) - self.torch_tensor = torch.ones(3, 3) - self.torch_param = torch.nn.Parameter(torch.ones(4, 4)) - -class PaddleModuleForTest(paddle.nn.Layer): - def __init__(self): - super(PaddleModuleForTest, self).__init__() - - self.paddle_fc1 = paddle.nn.Linear(10, 10) - self.paddle_softmax = paddle.nn.Softmax(0) - self.paddle_conv2d1 = paddle.nn.Conv2D(10, 10, 3) - self.paddle_tensor = paddle.ones((4, 4)) - - -@pytest.mark.torchpaddle -class TestTorchPaddleMixModule: - - def setup_method(self): - - self.model = MixModuleForTest() - self.torch_model = TorchModuleForTest() - self.paddle_model = PaddleModuleForTest() - - def test_to(self): - """ - 测试混合模型的to函数 - """ - - self.model.to("cuda") - self.torch_model.to("cuda") - self.paddle_model.to("gpu") - self.if_device_correct("cuda") - - self.model.to("cuda:2") - self.torch_model.to("cuda:2") - self.paddle_model.to("gpu:2") - self.if_device_correct("cuda:2") - - self.model.to("gpu:1") - self.torch_model.to("cuda:1") - self.paddle_model.to("gpu:1") - self.if_device_correct("cuda:1") - - self.model.to("cpu") - self.torch_model.to("cpu") - self.paddle_model.to("cpu") - self.if_device_correct("cpu") - - def test_train_eval(self): - """ - 测试train和eval函数 - """ - - self.model.eval() - self.if_training_correct(False) - - self.model.train() - self.if_training_correct(True) - - def test_parameters(self): - """ - 测试parameters()函数，由于初始化是随机的，目前仅比较得到结果的长度 - """ - mix_params = [] - params = [] - - for value in self.model.named_parameters(): - mix_params.append(value) - - for value in chain(self.torch_model.named_parameters(), self.paddle_model.named_parameters()): - params.append(value) - - assert len(params) == len(mix_params) - - def test_named_parameters(self): - """ - 测试named_parameters函数 - """ - - mix_param_names = [] - param_names = [] - - for name, value in self.model.named_parameters(): - mix_param_names.append(name) - - for name, value in chain(self.torch_model.named_parameters(), self.paddle_model.named_parameters()): - param_names.append(name) - - assert sorted(param_names) == sorted(mix_param_names) - - def test_torch_named_parameters(self): - """ - 测试对torch参数的提取 - """ - - mix_param_names = [] - param_names = [] - - for name, value in self.model.named_parameters(backend="torch"): - mix_param_names.append(name) - - for name, value in self.torch_model.named_parameters(): - param_names.append(name) - - assert sorted(param_names) == sorted(mix_param_names) - - def test_paddle_named_parameters(self): - """ - 测试对paddle参数的提取 - """ - - mix_param_names = [] - param_names = [] - - for name, value in self.model.named_parameters(backend="paddle"): - mix_param_names.append(name) - - for name, value in self.paddle_model.named_parameters(): - param_names.append(name) - - assert sorted(param_names) == sorted(mix_param_names) - - def test_torch_state_dict(self): - """ - 测试提取torch的state dict - """ - torch_dict = self.torch_model.state_dict() - mix_dict = self.model.state_dict(backend="torch") - - assert sorted(torch_dict.keys()) == sorted(mix_dict.keys()) - - def test_paddle_state_dict(self): - """ - 测试提取paddle的state dict - """ - paddle_dict = self.paddle_model.state_dict() - mix_dict = self.model.state_dict(backend="paddle") - - # TODO 测试程序会显示passed后显示paddle的异常退出信息 - assert sorted(paddle_dict.keys()) == sorted(mix_dict.keys()) - - def test_state_dict(self): - """ - 测试提取所有的state dict - """ - all_dict = self.torch_model.state_dict() - all_dict.update(self.paddle_model.state_dict()) - mix_dict = self.model.state_dict() - - # TODO 测试程序会显示passed后显示paddle的异常退出信息 - assert sorted(all_dict.keys()) == sorted(mix_dict.keys()) - - def test_load_state_dict(self): - """ - 测试load_state_dict函数 - """ - state_dict = self.model.state_dict() - - new_model = MixModuleForTest() - new_model.load_state_dict(state_dict) - new_state_dict = new_model.state_dict() - - for name, value in state_dict.items(): - state_dict[name] = value.tolist() - for name, value in new_state_dict.items(): - new_state_dict[name] = value.tolist() - - # self.assertDictEqual(state_dict, new_state_dict) - - def test_save_and_load_state_dict(self): - """ - 测试save_state_dict_to_file和load_state_dict_from_file函数 - """ - path = "model" - try: - self.model.save_state_dict_to_file(path) - new_model = MixModuleForTest() - new_model.load_state_dict_from_file(path) - - state_dict = self.model.state_dict() - new_state_dict = new_model.state_dict() - - for name, value in state_dict.items(): - state_dict[name] = value.tolist() - for name, value in new_state_dict.items(): - new_state_dict[name] = value.tolist() - - # self.assertDictEqual(state_dict, new_state_dict) - finally: - rank_zero_rm(path) - - def if_device_correct(self, device): - - - assert self.model.torch_fc1.weight.device == self.torch_model.torch_fc1.weight.device - assert self.model.torch_conv2d1.weight.device == self.torch_model.torch_fc1.bias.device - assert self.model.torch_conv2d1.bias.device == self.torch_model.torch_conv2d1.bias.device - assert self.model.torch_tensor.device == self.torch_model.torch_tensor.device - assert self.model.torch_param.device == self.torch_model.torch_param.device - - if device == "cpu": - assert self.model.paddle_fc1.weight.place.is_cpu_place() - assert self.model.paddle_fc1.bias.place.is_cpu_place() - assert self.model.paddle_conv2d1.weight.place.is_cpu_place() - assert self.model.paddle_conv2d1.bias.place.is_cpu_place() - assert self.model.paddle_tensor.place.is_cpu_place() - elif device.startswith("cuda"): - assert self.model.paddle_fc1.weight.place.is_gpu_place() - assert self.model.paddle_fc1.bias.place.is_gpu_place() - assert self.model.paddle_conv2d1.weight.place.is_gpu_place() - assert self.model.paddle_conv2d1.bias.place.is_gpu_place() - assert self.model.paddle_tensor.place.is_gpu_place() - - assert self.model.paddle_fc1.weight.place.gpu_device_id() == self.paddle_model.paddle_fc1.weight.place.gpu_device_id() - assert self.model.paddle_fc1.bias.place.gpu_device_id() == self.paddle_model.paddle_fc1.bias.place.gpu_device_id() - assert self.model.paddle_conv2d1.weight.place.gpu_device_id() == self.paddle_model.paddle_conv2d1.weight.place.gpu_device_id() - assert self.model.paddle_conv2d1.bias.place.gpu_device_id() == self.paddle_model.paddle_conv2d1.bias.place.gpu_device_id() - assert self.model.paddle_tensor.place.gpu_device_id() == self.paddle_model.paddle_tensor.place.gpu_device_id() - else: - raise NotImplementedError - - def if_training_correct(self, training): - - assert self.model.torch_fc1.training == training - assert self.model.torch_softmax.training == training - assert self.model.torch_conv2d1.training == training - - assert self.model.paddle_fc1.training == training - assert self.model.paddle_softmax.training == training - assert self.model.paddle_conv2d1.training == training - - -############################################################################ -# -# 测试在MNIST数据集上的表现 -# -############################################################################ - -class MNISTDataset(Dataset): - def __init__(self, dataset): - - self.dataset = [ - ( - np.array(img).astype('float32').reshape(-1), - label - ) for img, label in dataset - ] - - def __getitem__(self, idx): - return self.dataset[idx] - - def __len__(self): - return len(self.dataset) - -class MixMNISTModel(MixModule): - def __init__(self): - super(MixMNISTModel, self).__init__() - - self.fc1 = paddle.nn.Linear(784, 64) - self.fc2 = paddle.nn.Linear(64, 32) - self.fc3 = torch.nn.Linear(32, 10) - self.fc4 = torch.nn.Linear(10, 10) - - def forward(self, x): - - paddle_out = self.fc1(x) - paddle_out = self.fc2(paddle_out) - torch_in = paddle2torch(paddle_out) - torch_out = self.fc3(torch_in) - torch_out = self.fc4(torch_out) - - return torch_out - -@pytest.mark.torchpaddle -class TestMNIST: - - @classmethod - def setup_class(self): - - self.train_dataset = paddle.vision.datasets.MNIST(mode='train') - self.test_dataset = paddle.vision.datasets.MNIST(mode='test') - self.train_dataset = MNISTDataset(self.train_dataset) - - self.lr = 0.0003 - self.epochs = 20 - - self.dataloader = DataLoader(self.train_dataset, batch_size=100, shuffle=True) - - def setup_method(self): - - self.model = MixMNISTModel().to("cuda") - self.torch_loss_func = torch.nn.CrossEntropyLoss() - - self.torch_opt = torch.optim.Adam(self.model.parameters(backend="torch"), self.lr) - self.paddle_opt = paddle.optimizer.Adam(parameters=self.model.parameters(backend="paddle"), learning_rate=self.lr) - - def test_case1(self): - - # 开始训练 - for epoch in range(self.epochs): - epoch_loss, batch = 0, 0 - for batch, (img, label) in enumerate(self.dataloader): - - img = paddle.to_tensor(img).cuda() - torch_out = self.model(img) - label = torch.from_numpy(label.numpy()).reshape(-1) - loss = self.torch_loss_func(torch_out.cpu(), label) - epoch_loss += loss.item() - - loss.backward() - self.torch_opt.step() - self.paddle_opt.step() - self.torch_opt.zero_grad() - self.paddle_opt.clear_grad() - - else: - assert epoch_loss / (batch + 1) < 0.3 - - # 开始测试 - correct = 0 - for img, label in self.test_dataset: - - img = paddle.to_tensor(np.array(img).astype('float32').reshape(1, -1)) - torch_out = self.model(img) - res = torch_out.softmax(-1).argmax().item() - label = label.item() - if res == label: - correct += 1 - - acc = correct / len(self.test_dataset) - assert acc > 0.85 - -############################################################################ -# -# 测试在ERNIE中文数据集上的表现 -# -############################################################################ diff --git a/tests/modules/mix_modules/_test_utils.py b/tests/modules/mix_modules/_test_utils.py deleted file mode 100644 index ea7e55d7..00000000 --- a/tests/modules/mix_modules/_test_utils.py +++ /dev/null @@ -1,435 +0,0 @@ -import unittest -import os - -os.environ["log_silent"] = "1" -import torch -import paddle -import jittor - -from fastNLP.modules.mix_modules.utils import ( - paddle2torch, - torch2paddle, - jittor2torch, - torch2jittor, -) - -############################################################################ -# -# 测试paddle到torch的转换 -# -############################################################################ - -class Paddle2TorchTestCase(unittest.TestCase): - - def check_torch_tensor(self, tensor, device, requires_grad): - """ - 检查张量设备和梯度情况的工具函数 - """ - - assert isinstance(tensor, torch.Tensor) - assert tensor.device == torch.device(device) - assert tensor.requires_grad == requires_grad - - def test_gradient(self): - """ - 测试张量转换后的反向传播是否正确 - """ - - x = paddle.to_tensor([1.0, 2.0, 3.0, 4.0, 5.0], stop_gradient=False) - y = paddle2torch(x) - z = 3 * (y ** 2) - z.sum().backward() - assert y.grad.tolist() == [6, 12, 18, 24, 30] - - def test_tensor_transfer(self): - """ - 测试单个张量的设备和梯度转换是否正确 - """ - - paddle_tensor = paddle.rand((3, 4, 5)).cpu() - res = paddle2torch(paddle_tensor) - self.check_torch_tensor(res, "cpu", not paddle_tensor.stop_gradient) - - res = paddle2torch(paddle_tensor, target_device="cuda:2", no_gradient=None) - self.check_torch_tensor(res, "cuda:2", not paddle_tensor.stop_gradient) - - res = paddle2torch(paddle_tensor, target_device="cuda:1", no_gradient=True) - self.check_torch_tensor(res, "cuda:1", False) - - res = paddle2torch(paddle_tensor, target_device="cuda:1", no_gradient=False) - self.check_torch_tensor(res, "cuda:1", True) - - def test_list_transfer(self): - """ - 测试张量列表的转换 - """ - - paddle_list = [paddle.rand((6, 4, 2)).cuda(1) for i in range(10)] - res = paddle2torch(paddle_list) - assert isinstance(res, list) - for t in res: - self.check_torch_tensor(t, "cuda:1", False) - - res = paddle2torch(paddle_list, target_device="cpu", no_gradient=False) - assert isinstance(res, list) - for t in res: - self.check_torch_tensor(t, "cpu", True) - - def test_tensor_tuple_transfer(self): - """ - 测试张量元组的转换 - """ - - paddle_list = [paddle.rand((6, 4, 2)).cuda(1) for i in range(10)] - paddle_tuple = tuple(paddle_list) - res = paddle2torch(paddle_tuple) - assert isinstance(res, tuple) - for t in res: - self.check_torch_tensor(t, "cuda:1", False) - - def test_dict_transfer(self): - """ - 测试包含复杂结构的字典的转换 - """ - - paddle_dict = { - "tensor": paddle.rand((3, 4)).cuda(0), - "list": [paddle.rand((6, 4, 2)).cuda(0) for i in range(10)], - "dict":{ - "list": [paddle.rand((6, 4, 2)).cuda(0) for i in range(10)], - "tensor": paddle.rand((3, 4)).cuda(0) - }, - "int": 2, - "string": "test string" - } - res = paddle2torch(paddle_dict) - assert isinstance(res, dict) - self.check_torch_tensor(res["tensor"], "cuda:0", False) - assert isinstance(res["list"], list) - for t in res["list"]: - self.check_torch_tensor(t, "cuda:0", False) - assert isinstance(res["int"], int) - assert isinstance(res["string"], str) - assert isinstance(res["dict"], dict) - assert isinstance(res["dict"]["list"], list) - for t in res["dict"]["list"]: - self.check_torch_tensor(t, "cuda:0", False) - self.check_torch_tensor(res["dict"]["tensor"], "cuda:0", False) - - -############################################################################ -# -# 测试torch到paddle的转换 -# -############################################################################ - -class Torch2PaddleTestCase(unittest.TestCase): - - def check_paddle_tensor(self, tensor, device, stop_gradient): - """ - 检查得到的paddle张量设备和梯度情况的工具函数 - """ - - assert isinstance(tensor, paddle.Tensor) - if device == "cpu": - assert tensor.place.is_cpu_place() - elif device.startswith("gpu"): - paddle_device = paddle.device._convert_to_place(device) - assert tensor.place.is_gpu_place() - if hasattr(tensor.place, "gpu_device_id"): - # paddle中，有两种Place - # paddle.fluid.core.Place是创建Tensor时使用的类型 - # 有函数gpu_device_id获取设备 - assert tensor.place.gpu_device_id() == paddle_device.get_device_id() - else: - # 通过_convert_to_place得到的是paddle.CUDAPlace - # 通过get_device_id获取设备 - assert tensor.place.get_device_id() == paddle_device.get_device_id() - else: - raise NotImplementedError - assert tensor.stop_gradient == stop_gradient - - def test_gradient(self): - """ - 测试转换后梯度的反向传播 - """ - - x = torch.tensor([1.0, 2.0, 3.0, 4.0, 5.0], requires_grad=True) - y = torch2paddle(x) - z = 3 * (y ** 2) - z.sum().backward() - assert y.grad.tolist() == [6, 12, 18, 24, 30] - - def test_tensor_transfer(self): - """ - 测试单个张量的转换 - """ - - torch_tensor = torch.rand((3, 4, 5)) - res = torch2paddle(torch_tensor) - self.check_paddle_tensor(res, "cpu", True) - - res = torch2paddle(torch_tensor, target_device="gpu:2", no_gradient=None) - self.check_paddle_tensor(res, "gpu:2", True) - - res = torch2paddle(torch_tensor, target_device="gpu:2", no_gradient=True) - self.check_paddle_tensor(res, "gpu:2", True) - - res = torch2paddle(torch_tensor, target_device="gpu:2", no_gradient=False) - self.check_paddle_tensor(res, "gpu:2", False) - - def test_tensor_list_transfer(self): - """ - 测试张量列表的转换 - """ - - torch_list = [torch.rand(6, 4, 2) for i in range(10)] - res = torch2paddle(torch_list) - assert isinstance(res, list) - for t in res: - self.check_paddle_tensor(t, "cpu", True) - - res = torch2paddle(torch_list, target_device="gpu:1", no_gradient=False) - assert isinstance(res, list) - for t in res: - self.check_paddle_tensor(t, "gpu:1", False) - - def test_tensor_tuple_transfer(self): - """ - 测试张量元组的转换 - """ - - torch_list = [torch.rand(6, 4, 2) for i in range(10)] - torch_tuple = tuple(torch_list) - res = torch2paddle(torch_tuple, target_device="cpu") - assert isinstance(res, tuple) - for t in res: - self.check_paddle_tensor(t, "cpu", True) - - def test_dict_transfer(self): - """ - 测试复杂的字典结构的转换 - """ - - torch_dict = { - "tensor": torch.rand((3, 4)), - "list": [torch.rand(6, 4, 2) for i in range(10)], - "dict":{ - "list": [torch.rand(6, 4, 2) for i in range(10)], - "tensor": torch.rand((3, 4)) - }, - "int": 2, - "string": "test string" - } - res = torch2paddle(torch_dict) - assert isinstance(res, dict) - self.check_paddle_tensor(res["tensor"], "cpu", True) - assert isinstance(res["list"], list) - for t in res["list"]: - self.check_paddle_tensor(t, "cpu", True) - assert isinstance(res["int"], int) - assert isinstance(res["string"], str) - assert isinstance(res["dict"], dict) - assert isinstance(res["dict"]["list"], list) - for t in res["dict"]["list"]: - self.check_paddle_tensor(t, "cpu", True) - self.check_paddle_tensor(res["dict"]["tensor"], "cpu", True) - - -############################################################################ -# -# 测试jittor到torch的转换 -# -############################################################################ - -class Jittor2TorchTestCase(unittest.TestCase): - - def check_torch_tensor(self, tensor, device, requires_grad): - """ - 检查得到的torch张量的工具函数 - """ - - assert isinstance(tensor, torch.Tensor) - if device == "cpu": - assert not tensor.is_cuda - else: - assert tensor.device == torch.device(device) - assert tensor.requires_grad == requires_grad - - def test_var_transfer(self): - """ - 测试单个Jittor Var的转换 - """ - - jittor_var = jittor.rand((3, 4, 5)) - res = jittor2torch(jittor_var) - self.check_torch_tensor(res, "cpu", True) - - res = jittor2torch(jittor_var, target_device="cuda:2", no_gradient=None) - self.check_torch_tensor(res, "cuda:2", True) - - res = jittor2torch(jittor_var, target_device="cuda:2", no_gradient=True) - self.check_torch_tensor(res, "cuda:2", False) - - res = jittor2torch(jittor_var, target_device="cuda:2", no_gradient=False) - self.check_torch_tensor(res, "cuda:2", True) - - def test_var_list_transfer(self): - """ - 测试Jittor列表的转换 - """ - - jittor_list = [jittor.rand((6, 4, 2)) for i in range(10)] - res = jittor2torch(jittor_list) - assert isinstance(res, list) - for t in res: - self.check_torch_tensor(t, "cpu", True) - - res = jittor2torch(jittor_list, target_device="cuda:1", no_gradient=False) - assert isinstance(res, list) - for t in res: - self.check_torch_tensor(t, "cuda:1", True) - - def test_var_tuple_transfer(self): - """ - 测试Jittor变量元组的转换 - """ - - jittor_list = [jittor.rand((6, 4, 2)) for i in range(10)] - jittor_tuple = tuple(jittor_list) - res = jittor2torch(jittor_tuple, target_device="cpu") - assert isinstance(res, tuple) - for t in res: - self.check_torch_tensor(t, "cpu", True) - - def test_dict_transfer(self): - """ - 测试字典结构的转换 - """ - - jittor_dict = { - "tensor": jittor.rand((3, 4)), - "list": [jittor.rand(6, 4, 2) for i in range(10)], - "dict":{ - "list": [jittor.rand(6, 4, 2) for i in range(10)], - "tensor": jittor.rand((3, 4)) - }, - "int": 2, - "string": "test string" - } - res = jittor2torch(jittor_dict) - assert isinstance(res, dict) - self.check_torch_tensor(res["tensor"], "cpu", True) - assert isinstance(res["list"], list) - for t in res["list"]: - self.check_torch_tensor(t, "cpu", True) - assert isinstance(res["int"], int) - assert isinstance(res["string"], str) - assert isinstance(res["dict"], dict) - assert isinstance(res["dict"]["list"], list) - for t in res["dict"]["list"]: - self.check_torch_tensor(t, "cpu", True) - self.check_torch_tensor(res["dict"]["tensor"], "cpu", True) - - -############################################################################ -# -# 测试torch到jittor的转换 -# -############################################################################ - -class Torch2JittorTestCase(unittest.TestCase): - - def check_jittor_var(self, var, requires_grad): - """ - 检查得到的Jittor Var梯度情况的工具函数 - """ - - assert isinstance(var, jittor.Var) - assert var.requires_grad == requires_grad - - def test_gradient(self): - """ - 测试反向传播的梯度 - """ - - x = torch.tensor([1.0, 2.0, 3.0, 4.0, 5.0], requires_grad=True) - y = torch2jittor(x) - z = 3 * (y ** 2) - grad = jittor.grad(z, y) - assert grad.tolist() == [6.0, 12.0, 18.0, 24.0, 30.0] - - def test_tensor_transfer(self): - """ - 测试单个张量转换为Jittor - """ - - torch_tensor = torch.rand((3, 4, 5)) - res = torch2jittor(torch_tensor) - self.check_jittor_var(res, False) - - res = torch2jittor(torch_tensor, no_gradient=None) - self.check_jittor_var(res, False) - - res = torch2jittor(torch_tensor, no_gradient=True) - self.check_jittor_var(res, False) - - res = torch2jittor(torch_tensor, no_gradient=False) - self.check_jittor_var(res, True) - - def test_tensor_list_transfer(self): - """ - 测试张量列表的转换 - """ - - torch_list = [torch.rand((6, 4, 2)) for i in range(10)] - res = torch2jittor(torch_list) - assert isinstance(res, list) - for t in res: - self.check_jittor_var(t, False) - - res = torch2jittor(torch_list, no_gradient=False) - assert isinstance(res, list) - for t in res: - self.check_jittor_var(t, True) - - def test_tensor_tuple_transfer(self): - """ - 测试张量元组的转换 - """ - - torch_list = [torch.rand((6, 4, 2)) for i in range(10)] - torch_tuple = tuple(torch_list) - res = torch2jittor(torch_tuple) - assert isinstance(res, tuple) - for t in res: - self.check_jittor_var(t, False) - - def test_dict_transfer(self): - """ - 测试字典结构的转换 - """ - - torch_dict = { - "tensor": torch.rand((3, 4)), - "list": [torch.rand(6, 4, 2) for i in range(10)], - "dict":{ - "list": [torch.rand(6, 4, 2) for i in range(10)], - "tensor": torch.rand((3, 4)) - }, - "int": 2, - "string": "test string" - } - res = torch2jittor(torch_dict) - assert isinstance(res, dict) - self.check_jittor_var(res["tensor"], False) - assert isinstance(res["list"], list) - for t in res["list"]: - self.check_jittor_var(t, False) - assert isinstance(res["int"], int) - assert isinstance(res["string"], str) - assert isinstance(res["dict"], dict) - assert isinstance(res["dict"]["list"], list) - for t in res["dict"]["list"]: - self.check_jittor_var(t, False) - self.check_jittor_var(res["dict"]["tensor"], False)