@@ -205,6 +205,8 @@ class TopkSaver(ResultsMonitor, Saver): | |||||
def __init__(self, topk:int=0, monitor:str=None, larger_better:bool=True, folder:str=None, save_object:str='model', | def __init__(self, topk:int=0, monitor:str=None, larger_better:bool=True, folder:str=None, save_object:str='model', | ||||
only_state_dict:bool=True, model_save_fn:Callable=None, save_evaluate_results:bool=True, | only_state_dict:bool=True, model_save_fn:Callable=None, save_evaluate_results:bool=True, | ||||
**kwargs): | **kwargs): | ||||
if topk is None: | |||||
topk = 0 | |||||
ResultsMonitor.__init__(self, monitor, larger_better) | ResultsMonitor.__init__(self, monitor, larger_better) | ||||
Saver.__init__(self, folder, save_object, only_state_dict, model_save_fn, **kwargs) | Saver.__init__(self, folder, save_object, only_state_dict, model_save_fn, **kwargs) | ||||
@@ -134,7 +134,11 @@ class JittorTensorPadder(Padder): | |||||
f"it must have tolist() method.") | f"it must have tolist() method.") | ||||
shapes = [field.shape for field in batch_field] | shapes = [field.shape for field in batch_field] | ||||
max_shape = [len(batch_field)] + [max(*_) for _ in zip(*shapes)] | |||||
if len(batch_field) < 2: | |||||
max_shape = [len(batch_field)] + list(shapes[0]) | |||||
else: | |||||
max_shape = [len(batch_field)] + [max(*_) for _ in zip(*shapes)] | |||||
# if dtype is not None: | # if dtype is not None: | ||||
# tensor = jittor.full(max_shape, pad_val, dtype=dtype) | # tensor = jittor.full(max_shape, pad_val, dtype=dtype) | ||||
# else: | # else: | ||||
@@ -97,7 +97,11 @@ class NumpyTensorPadder(Padder): | |||||
f"it must have tolist() method.") | f"it must have tolist() method.") | ||||
shapes = [field.shape for field in batch_field] | shapes = [field.shape for field in batch_field] | ||||
max_shape = [len(batch_field)] + [max(*_) for _ in zip(*shapes)] | |||||
if len(batch_field) < 2: | |||||
max_shape = [len(batch_field)] + list(shapes[0]) | |||||
else: | |||||
max_shape = [len(batch_field)] + [max(*_) for _ in zip(*shapes)] | |||||
array = np.full(max_shape, fill_value=pad_val, dtype=dtype) | array = np.full(max_shape, fill_value=pad_val, dtype=dtype) | ||||
for i, field in enumerate(batch_field): | for i, field in enumerate(batch_field): | ||||
slices = (i, ) + tuple(slice(0, s) for s in shapes[i]) | slices = (i, ) + tuple(slice(0, s) for s in shapes[i]) | ||||
@@ -140,7 +140,11 @@ class PaddleTensorPadder(Padder): | |||||
f"it must have tolist() method.") | f"it must have tolist() method.") | ||||
shapes = [field.shape for field in batch_field] | shapes = [field.shape for field in batch_field] | ||||
max_shape = [len(batch_field)] + [max(*_) for _ in zip(*shapes)] | |||||
if len(batch_field) < 2: | |||||
max_shape = [len(batch_field)] + list(shapes[0]) | |||||
else: | |||||
max_shape = [len(batch_field)] + [max(*_) for _ in zip(*shapes)] | |||||
if isinstance(batch_field[0], paddle.Tensor): | if isinstance(batch_field[0], paddle.Tensor): | ||||
array = paddle.full(max_shape, fill_value=pad_val, dtype=dtype) | array = paddle.full(max_shape, fill_value=pad_val, dtype=dtype) | ||||
else: | else: | ||||
@@ -132,7 +132,11 @@ class TorchTensorPadder(Padder): | |||||
f"it must have tolist() method.") | f"it must have tolist() method.") | ||||
shapes = [field.shape for field in batch_field] | shapes = [field.shape for field in batch_field] | ||||
max_shape = [len(batch_field)] + [max(*_) for _ in zip(*shapes)] | |||||
if len(batch_field) < 2: | |||||
max_shape = [len(batch_field)] + list(shapes[0]) | |||||
else: | |||||
max_shape = [len(batch_field)] + [max(*_) for _ in zip(*shapes)] | |||||
tensor = torch.full(max_shape, fill_value=pad_val, dtype=dtype, device=device) | tensor = torch.full(max_shape, fill_value=pad_val, dtype=dtype, device=device) | ||||
for i, field in enumerate(batch_field): | for i, field in enumerate(batch_field): | ||||
slices = (i, ) + tuple(slice(0, s) for s in shapes[i]) | slices = (i, ) + tuple(slice(0, s) for s in shapes[i]) | ||||
@@ -32,7 +32,11 @@ def get_shape(batch_field:List, shape=None): | |||||
if isinstance(batch_field[0], Sequence): | if isinstance(batch_field[0], Sequence): | ||||
for _field in batch_field: | for _field in batch_field: | ||||
shapes.append(get_shape(_field, _shape)) | shapes.append(get_shape(_field, _shape)) | ||||
max_shape = [max(_) for _ in zip(*shapes)] | |||||
if len(shapes) == 1: | |||||
max_shape = shapes[0] | |||||
else: | |||||
max_shape = [max(_) for _ in zip(*shapes)] | |||||
return max_shape | return max_shape | ||||
except IndexError: # 空的shape | except IndexError: # 空的shape | ||||
pass | pass | ||||
@@ -618,9 +618,9 @@ class Trainer(TrainerEventTrigger): | |||||
if not catch_KeyboardInterrupt: | if not catch_KeyboardInterrupt: | ||||
raise e | raise e | ||||
except RuntimeError as e: | except RuntimeError as e: | ||||
if 'torch' in self.driver_name.lower(): # 如果是 torch ,需要检测一下 find_unused_parameters | |||||
if 'torch' in self.driver_name.lower() and len(e.args) > 0: # 如果是 torch ,需要检测一下 find_unused_parameters | |||||
if 'find_unused_parameters' in e.args[0]: | if 'find_unused_parameters' in e.args[0]: | ||||
logger.error("You may need to pass `torch_ddp_kwargs={'find_unused_parameters': True}` in the " | |||||
logger.error("You may need to pass `torch_kwargs={'ddp_kwargs':{'find_unused_parameters': True}}` in the " | |||||
"Trainer initialization to avoid this error.") | "Trainer initialization to avoid this error.") | ||||
self.driver.on_exception() | self.driver.on_exception() | ||||
self.on_exception(e) | self.on_exception(e) | ||||
@@ -249,7 +249,7 @@ class PaddleDataLoader(DataLoader): | |||||
def prepare_paddle_dataloader(ds_or_db, feed_list=None, places=None, | def prepare_paddle_dataloader(ds_or_db, feed_list=None, places=None, | ||||
return_list: bool = True, | return_list: bool = True, | ||||
batch_sampler: Union["Sampler[Sequence[int]]", ReproducibleBatchSampler] = None, | batch_sampler: Union["Sampler[Sequence[int]]", ReproducibleBatchSampler] = None, | ||||
train_batch_size: int = 1, shuffle: bool = False, | |||||
batch_size: int = 1, shuffle: bool = False, | |||||
drop_last: bool = False, collate_fn: Union[Callable, str, None] = 'auto', | drop_last: bool = False, collate_fn: Union[Callable, str, None] = 'auto', | ||||
num_workers: int = 0, use_buffer_reader: bool = True, | num_workers: int = 0, use_buffer_reader: bool = True, | ||||
use_shared_memory: bool = True, timeout: int = 0, | use_shared_memory: bool = True, timeout: int = 0, | ||||
@@ -259,7 +259,7 @@ def prepare_paddle_dataloader(ds_or_db, feed_list=None, places=None, | |||||
from fastNLP.io.data_bundle import DataBundle | from fastNLP.io.data_bundle import DataBundle | ||||
if isinstance(ds_or_db, Dataset): | if isinstance(ds_or_db, Dataset): | ||||
dl = PaddleDataLoader(ds_or_db, feed_list=feed_list, places=places, return_list=return_list, | dl = PaddleDataLoader(ds_or_db, feed_list=feed_list, places=places, return_list=return_list, | ||||
batch_sampler=batch_sampler, batch_size=train_batch_size, shuffle=shuffle, | |||||
batch_sampler=batch_sampler, batch_size=batch_size, shuffle=shuffle, | |||||
drop_last=drop_last, collate_fn=collate_fn, num_workers=num_workers, | drop_last=drop_last, collate_fn=collate_fn, num_workers=num_workers, | ||||
use_shared_memory=use_shared_memory, use_buffer_reader=use_buffer_reader, | use_shared_memory=use_shared_memory, use_buffer_reader=use_buffer_reader, | ||||
timeout=timeout, worker_init_fn=worker_init_fn, persistent_workers=persistent_workers) | timeout=timeout, worker_init_fn=worker_init_fn, persistent_workers=persistent_workers) | ||||
@@ -270,7 +270,7 @@ def prepare_paddle_dataloader(ds_or_db, feed_list=None, places=None, | |||||
if 'train' in name: | if 'train' in name: | ||||
dl_bundle[name] = PaddleDataLoader(ds, feed_list=feed_list, places=places, | dl_bundle[name] = PaddleDataLoader(ds, feed_list=feed_list, places=places, | ||||
return_list=return_list, | return_list=return_list, | ||||
batch_sampler=batch_sampler, batch_size=train_batch_size, | |||||
batch_sampler=batch_sampler, batch_size=batch_size, | |||||
shuffle=shuffle, | shuffle=shuffle, | ||||
drop_last=drop_last, collate_fn=collate_fn, num_workers=num_workers, | drop_last=drop_last, collate_fn=collate_fn, num_workers=num_workers, | ||||
use_shared_memory=use_shared_memory, | use_shared_memory=use_shared_memory, | ||||
@@ -292,7 +292,7 @@ def prepare_paddle_dataloader(ds_or_db, feed_list=None, places=None, | |||||
ds_seq = [] | ds_seq = [] | ||||
for ds in ds_or_db: | for ds in ds_or_db: | ||||
dl = PaddleDataLoader(ds, feed_list=feed_list, places=places, return_list=return_list, | dl = PaddleDataLoader(ds, feed_list=feed_list, places=places, return_list=return_list, | ||||
batch_sampler=batch_sampler, batch_size=train_batch_size, shuffle=shuffle, | |||||
batch_sampler=batch_sampler, batch_size=batch_size, shuffle=shuffle, | |||||
drop_last=drop_last, collate_fn=collate_fn, num_workers=num_workers, | drop_last=drop_last, collate_fn=collate_fn, num_workers=num_workers, | ||||
use_shared_memory=use_shared_memory, use_buffer_reader=use_buffer_reader, | use_shared_memory=use_shared_memory, use_buffer_reader=use_buffer_reader, | ||||
timeout=timeout, worker_init_fn=worker_init_fn, persistent_workers=persistent_workers) | timeout=timeout, worker_init_fn=worker_init_fn, persistent_workers=persistent_workers) | ||||
@@ -304,7 +304,7 @@ def prepare_paddle_dataloader(ds_or_db, feed_list=None, places=None, | |||||
for name, ds in ds_or_db.items(): | for name, ds in ds_or_db.items(): | ||||
if 'train' in name: | if 'train' in name: | ||||
dl = PaddleDataLoader(ds, feed_list=feed_list, places=places, return_list=return_list, | dl = PaddleDataLoader(ds, feed_list=feed_list, places=places, return_list=return_list, | ||||
batch_sampler=batch_sampler, batch_size=train_batch_size, shuffle=shuffle, | |||||
batch_sampler=batch_sampler, batch_size=batch_size, shuffle=shuffle, | |||||
drop_last=drop_last, collate_fn=collate_fn, num_workers=num_workers, | drop_last=drop_last, collate_fn=collate_fn, num_workers=num_workers, | ||||
use_shared_memory=use_shared_memory, use_buffer_reader=use_buffer_reader, | use_shared_memory=use_shared_memory, use_buffer_reader=use_buffer_reader, | ||||
timeout=timeout, worker_init_fn=worker_init_fn, | timeout=timeout, worker_init_fn=worker_init_fn, | ||||
@@ -109,6 +109,9 @@ def _get_backend(): | |||||
if len(available_backends) == 1: | if len(available_backends) == 1: | ||||
backend = available_backends.pop() | backend = available_backends.pop() | ||||
logger.debug(f"Get Dataloader backend:{backend} from sys.modules.") | logger.debug(f"Get Dataloader backend:{backend} from sys.modules.") | ||||
elif len(available_backends) > 1: | |||||
raise RuntimeError("Fail to detect dataloader backend automatically, because multiple backends:" | |||||
f"{available_backends} has been imported.") | |||||
else: | else: | ||||
raise RuntimeError("Fail to detect dataloader backend automatically, please set it manually.") | raise RuntimeError("Fail to detect dataloader backend automatically, please set it manually.") | ||||
return backend | return backend |
@@ -78,9 +78,14 @@ class TorchDataLoader(DataLoader): | |||||
if not isinstance(dataset, _FDataSet): | if not isinstance(dataset, _FDataSet): | ||||
dataset = _FDataSet(dataset) | dataset = _FDataSet(dataset) | ||||
if sampler is None and batch_sampler is None: | |||||
if batch_sampler is not None: | |||||
batch_size = 1 | |||||
shuffle = False | |||||
sampler = None | |||||
elif sampler is None: | |||||
sampler = RandomSampler(dataset, shuffle=shuffle) | sampler = RandomSampler(dataset, shuffle=shuffle) | ||||
shuffle = False | shuffle = False | ||||
if isinstance(collate_fn, str): | if isinstance(collate_fn, str): | ||||
if collate_fn == 'auto': | if collate_fn == 'auto': | ||||
if isinstance(dataset.dataset, DataSet): # 使用了 fastnlp dataset | if isinstance(dataset.dataset, DataSet): # 使用了 fastnlp dataset | ||||
@@ -179,7 +184,7 @@ class TorchDataLoader(DataLoader): | |||||
def prepare_torch_dataloader(ds_or_db, | def prepare_torch_dataloader(ds_or_db, | ||||
train_batch_size: int = 16, | |||||
batch_size: int = 16, | |||||
shuffle: bool = False, | shuffle: bool = False, | ||||
sampler: Union["Sampler[int]", ReproducibleSampler, UnrepeatedSampler] = None, | sampler: Union["Sampler[int]", ReproducibleSampler, UnrepeatedSampler] = None, | ||||
batch_sampler: Union["Sampler[Sequence[int]]", ReproducibleBatchSampler] = None, | batch_sampler: Union["Sampler[Sequence[int]]", ReproducibleBatchSampler] = None, | ||||
@@ -215,7 +220,7 @@ def prepare_torch_dataloader(ds_or_db, | |||||
from fastNLP.io import DataBundle | from fastNLP.io import DataBundle | ||||
if isinstance(ds_or_db, DataSet): | if isinstance(ds_or_db, DataSet): | ||||
dl = TorchDataLoader(dataset=ds_or_db, batch_size=train_batch_size, | |||||
dl = TorchDataLoader(dataset=ds_or_db, batch_size=batch_size, | |||||
shuffle=shuffle, sampler=sampler, batch_sampler=batch_sampler, | shuffle=shuffle, sampler=sampler, batch_sampler=batch_sampler, | ||||
num_workers=num_workers, collate_fn=collate_fn, pin_memory=pin_memory, | num_workers=num_workers, collate_fn=collate_fn, pin_memory=pin_memory, | ||||
drop_last=drop_last, timeout=timeout, worker_init_fn=worker_init_fn, | drop_last=drop_last, timeout=timeout, worker_init_fn=worker_init_fn, | ||||
@@ -228,7 +233,7 @@ def prepare_torch_dataloader(ds_or_db, | |||||
dl_bundle = {} | dl_bundle = {} | ||||
for name, ds in ds_or_db.iter_datasets(): | for name, ds in ds_or_db.iter_datasets(): | ||||
if 'train' in name: | if 'train' in name: | ||||
dl_bundle[name] = TorchDataLoader(dataset=ds, batch_size=train_batch_size, | |||||
dl_bundle[name] = TorchDataLoader(dataset=ds, batch_size=batch_size, | |||||
shuffle=shuffle, sampler=sampler, batch_sampler=batch_sampler, | shuffle=shuffle, sampler=sampler, batch_sampler=batch_sampler, | ||||
num_workers=num_workers, collate_fn=collate_fn, pin_memory=pin_memory, | num_workers=num_workers, collate_fn=collate_fn, pin_memory=pin_memory, | ||||
drop_last=drop_last, timeout=timeout, worker_init_fn=worker_init_fn, | drop_last=drop_last, timeout=timeout, worker_init_fn=worker_init_fn, | ||||
@@ -237,7 +242,7 @@ def prepare_torch_dataloader(ds_or_db, | |||||
persistent_workers=persistent_workers, | persistent_workers=persistent_workers, | ||||
) | ) | ||||
else: | else: | ||||
dl_bundle[name] = TorchDataLoader(dataset=ds, batch_size=non_train_batch_size if non_train_batch_size else train_batch_size, | |||||
dl_bundle[name] = TorchDataLoader(dataset=ds, batch_size=non_train_batch_size if non_train_batch_size else batch_size, | |||||
shuffle=shuffle, sampler=non_train_sampler if non_train_sampler else sampler, | shuffle=shuffle, sampler=non_train_sampler if non_train_sampler else sampler, | ||||
batch_sampler=batch_sampler, | batch_sampler=batch_sampler, | ||||
num_workers=num_workers, collate_fn=collate_fn, pin_memory=pin_memory, | num_workers=num_workers, collate_fn=collate_fn, pin_memory=pin_memory, | ||||
@@ -252,10 +257,10 @@ def prepare_torch_dataloader(ds_or_db, | |||||
dl_bundle = [] | dl_bundle = [] | ||||
for idx, ds in enumerate(ds_or_db): | for idx, ds in enumerate(ds_or_db): | ||||
if idx > 0: | if idx > 0: | ||||
train_batch_size = non_train_batch_size if non_train_batch_size else train_batch_size | |||||
batch_size = non_train_batch_size if non_train_batch_size else batch_size | |||||
sampler = non_train_sampler if non_train_sampler else sampler | sampler = non_train_sampler if non_train_sampler else sampler | ||||
dl_bundle.append( | dl_bundle.append( | ||||
TorchDataLoader(dataset=ds, batch_size=train_batch_size, | |||||
TorchDataLoader(dataset=ds, batch_size=batch_size, | |||||
shuffle=shuffle, sampler=sampler, batch_sampler=batch_sampler, | shuffle=shuffle, sampler=sampler, batch_sampler=batch_sampler, | ||||
num_workers=num_workers, collate_fn=collate_fn, pin_memory=pin_memory, | num_workers=num_workers, collate_fn=collate_fn, pin_memory=pin_memory, | ||||
drop_last=drop_last, timeout=timeout, worker_init_fn=worker_init_fn, | drop_last=drop_last, timeout=timeout, worker_init_fn=worker_init_fn, | ||||
@@ -269,7 +274,7 @@ def prepare_torch_dataloader(ds_or_db, | |||||
dl_bundle = {} | dl_bundle = {} | ||||
for name, ds in ds_or_db.items(): | for name, ds in ds_or_db.items(): | ||||
if 'train' in name: | if 'train' in name: | ||||
dl_bundle[name] = TorchDataLoader(dataset=ds, batch_size=train_batch_size, | |||||
dl_bundle[name] = TorchDataLoader(dataset=ds, batch_size=batch_size, | |||||
shuffle=shuffle, sampler=sampler, batch_sampler=batch_sampler, | shuffle=shuffle, sampler=sampler, batch_sampler=batch_sampler, | ||||
num_workers=num_workers, collate_fn=collate_fn, pin_memory=pin_memory, | num_workers=num_workers, collate_fn=collate_fn, pin_memory=pin_memory, | ||||
drop_last=drop_last, timeout=timeout, worker_init_fn=worker_init_fn, | drop_last=drop_last, timeout=timeout, worker_init_fn=worker_init_fn, | ||||
@@ -278,7 +283,7 @@ def prepare_torch_dataloader(ds_or_db, | |||||
persistent_workers=persistent_workers, | persistent_workers=persistent_workers, | ||||
) | ) | ||||
else: | else: | ||||
dl_bundle[name] = TorchDataLoader(dataset=ds, batch_size=non_train_batch_size if non_train_batch_size else train_batch_size, | |||||
dl_bundle[name] = TorchDataLoader(dataset=ds, batch_size=non_train_batch_size if non_train_batch_size else batch_size, | |||||
shuffle=shuffle, sampler=non_train_sampler if non_train_sampler else sampler, | shuffle=shuffle, sampler=non_train_sampler if non_train_sampler else sampler, | ||||
batch_sampler=batch_sampler, | batch_sampler=batch_sampler, | ||||
num_workers=num_workers, collate_fn=collate_fn, pin_memory=pin_memory, | num_workers=num_workers, collate_fn=collate_fn, pin_memory=pin_memory, | ||||
@@ -497,7 +497,7 @@ class DataSet: | |||||
:param progress_desc: 进度条的描述字符,默认为'Main | :param progress_desc: 进度条的描述字符,默认为'Main | ||||
""" | """ | ||||
if isinstance(func, LambdaType) and num_proc>1 and func.__name__ == "<lambda>": | if isinstance(func, LambdaType) and num_proc>1 and func.__name__ == "<lambda>": | ||||
raise ("Lambda function does not support multiple processes, please set `num_proc=0`.") | |||||
raise TypeError("Lambda function does not support multiple processes, please set `num_proc=0`.") | |||||
if num_proc>1 and sys.platform in ('win32', 'msys', 'cygwin'): | if num_proc>1 and sys.platform in ('win32', 'msys', 'cygwin'): | ||||
raise RuntimeError("Your platform does not support multiprocessing with fork, please set `num_proc=0`") | raise RuntimeError("Your platform does not support multiprocessing with fork, please set `num_proc=0`") | ||||
@@ -6,7 +6,7 @@ __all__ = [ | |||||
import math | import math | ||||
from copy import deepcopy | from copy import deepcopy | ||||
from typing import Dict, Union, List | |||||
from typing import Dict, Union, List, Sequence | |||||
from itertools import chain | from itertools import chain | ||||
import numpy as np | import numpy as np | ||||
@@ -390,17 +390,20 @@ class BucketedBatchSampler(ReproducibleBatchSampler): | |||||
length = dataset.get_field(length).content | length = dataset.get_field(length).content | ||||
if not isinstance(length[0], int): | if not isinstance(length[0], int): | ||||
length = list(map(len, length)) | length = list(map(len, length)) | ||||
self.length = np.array(length, dtype=int) | |||||
self.sorted_indices = np.argsort(self.length)[::-1] # 按长度从高到低排序的 | |||||
else: | else: | ||||
types = set(map(type, length)) | |||||
assert isinstance(length, list) and len(types)==1 and types.pop()==int, \ | |||||
"When the dataset is not fastNLP.DataSet, the length parameter can only be List[int]" | |||||
try: | |||||
self.length = np.array(length, dtype=int) | |||||
self.sorted_indices = np.argsort(length)[::-1] | |||||
except BaseException as e: | |||||
logger.error(f"Cannot use {self.__class__.__name__} as length, since it is not sortable.") | |||||
assert len(length) == len(dataset), f"The length of `dataset`({len(dataset)}) and " \ | assert len(length) == len(dataset), f"The length of `dataset`({len(dataset)}) and " \ | ||||
f"`length`({len(length)}) should be equal." | f"`length`({len(length)}) should be equal." | ||||
assert len(self.sorted_indices) == len(dataset), "The indices and dataset should have equal length." | |||||
self.dataset = dataset | self.dataset = dataset | ||||
self.length = np.array(length, dtype=int) # 按照长到短排列的序号。 | |||||
self.sorted_indices = np.argsort(self.length)[::-1] # 按长度从高到低排序的 | |||||
self.batch_size = batch_size | self.batch_size = batch_size | ||||
self.num_batch_per_bucket = num_batch_per_bucket | self.num_batch_per_bucket = num_batch_per_bucket | ||||
@@ -5,7 +5,7 @@ __all__ = [ | |||||
"SequentialSampler" | "SequentialSampler" | ||||
] | ] | ||||
from typing import Dict, List, Union | |||||
from typing import Dict, List, Union, Sequence | |||||
import math | import math | ||||
import numpy as np | import numpy as np | ||||
@@ -305,12 +305,18 @@ class SortedSampler(SequentialSampler): | |||||
length = dataset.get_field(length).content | length = dataset.get_field(length).content | ||||
if not isinstance(length[0], int): | if not isinstance(length[0], int): | ||||
length = list(map(len, length)) | length = list(map(len, length)) | ||||
self.length = np.array(length, dtype=int) | |||||
self.sorted_indices = np.argsort(self.length)[::-1] # 按长度从高到低排序的 | |||||
else: | else: | ||||
types = set(map(type, length)) | |||||
assert isinstance(length, list) and len(types)==1 and types.pop()==int, \ | |||||
"When the dataset is not fastNLP.DataSet, the length parameter can only be List[int]" | |||||
assert len(length) == len(dataset), "The length of `data` and `length` should be equal." | |||||
try: | |||||
self.length = np.array(length, dtype=int) | |||||
self.sorted_indices = np.argsort(length)[::-1] | |||||
except BaseException as e: | |||||
logger.error(f"Cannot use {self.__class__.__name__} as length, since it is not sortable.") | |||||
assert len(length) == len(dataset), f"The length of `dataset`({len(dataset)}) and " \ | |||||
f"`length`({len(length)}) should be equal." | |||||
assert len(self.sorted_indices) == len(dataset), "The indices and dataset should have equal length." | |||||
self.length = np.array(length, dtype=int) # 按照长到短排列的序号。 | self.length = np.array(length, dtype=int) # 按照长到短排列的序号。 | ||||
self.sorted_indices = np.argsort(self.length)[::-1].tolist() # 按长度从高到低排序的 | self.sorted_indices = np.argsort(self.length)[::-1].tolist() # 按长度从高到低排序的 | ||||
@@ -184,7 +184,7 @@ class TestDataSetMethods: | |||||
ds.apply(lambda ins: len(ins["y"]), new_field_name="y", progress_bar=None) | ds.apply(lambda ins: len(ins["y"]), new_field_name="y", progress_bar=None) | ||||
assert ds.field_arrays["y"].content[0] == 2 | assert ds.field_arrays["y"].content[0] == 2 | ||||
res = ds.apply(lambda ins: len(ins["x"]), num_proc=2, progress_desc="len") | |||||
res = ds.apply(lambda ins: len(ins["x"]), num_proc=0, progress_desc="len") | |||||
assert (isinstance(res, list) and len(res) > 0) == True | assert (isinstance(res, list) and len(res) > 0) == True | ||||
assert res[0] == 4 | assert res[0] == 4 | ||||
@@ -377,7 +377,7 @@ class TestDataSetMethods: | |||||
def test_apply_proc(self): | def test_apply_proc(self): | ||||
data = DataSet({'x': ['xxxxas1w xw zxw xz', 'xxxxas1w xw zxw xz'] * 100, 'y': [0, 1] * 100}) | data = DataSet({'x': ['xxxxas1w xw zxw xz', 'xxxxas1w xw zxw xz'] * 100, 'y': [0, 1] * 100}) | ||||
data.apply_field(lambda x: len(x), field_name='x', new_field_name='len_x', num_proc=2) | |||||
data.apply_field(lambda x: len(x), field_name='x', new_field_name='len_x', num_proc=0) | |||||
class TestFieldArrayInit: | class TestFieldArrayInit: | ||||
@@ -87,16 +87,10 @@ class TestPaddleMoveDataToDevice: | |||||
""" | """ | ||||
paddle_tensor = paddle.rand((3, 4, 5)).cpu() | paddle_tensor = paddle.rand((3, 4, 5)).cpu() | ||||
res = paddle_move_data_to_device(paddle_tensor, device=None, data_device=None) | |||||
res = paddle_move_data_to_device(paddle_tensor, device=None) | |||||
self.check_cpu(res) | self.check_cpu(res) | ||||
res = paddle_move_data_to_device(paddle_tensor, device="gpu:0", data_device=None) | |||||
self.check_gpu(res, 0) | |||||
res = paddle_move_data_to_device(paddle_tensor, device="gpu:0", data_device="cpu") | |||||
self.check_gpu(res, 0) | |||||
res = paddle_move_data_to_device(paddle_tensor, device=None, data_device="gpu:0") | |||||
res = paddle_move_data_to_device(paddle_tensor, device="gpu:0") | |||||
self.check_gpu(res, 0) | self.check_gpu(res, 0) | ||||
def test_list_transfer(self): | def test_list_transfer(self): | ||||
@@ -106,12 +100,12 @@ class TestPaddleMoveDataToDevice: | |||||
paddle_list = [paddle.rand((6, 4, 2)) for i in range(10)] | paddle_list = [paddle.rand((6, 4, 2)) for i in range(10)] | ||||
res = paddle_move_data_to_device(paddle_list, device="cpu", data_device="gpu:1") | |||||
res = paddle_move_data_to_device(paddle_list, device="cpu") | |||||
assert isinstance(res, list) | assert isinstance(res, list) | ||||
for r in res: | for r in res: | ||||
self.check_cpu(r) | self.check_cpu(r) | ||||
res = paddle_move_data_to_device(paddle_list, device="gpu:0", data_device=None) | |||||
res = paddle_move_data_to_device(paddle_list, device="gpu:0") | |||||
assert isinstance(res, list) | assert isinstance(res, list) | ||||
for r in res: | for r in res: | ||||
self.check_gpu(r, 0) | self.check_gpu(r, 0) | ||||
@@ -124,12 +118,12 @@ class TestPaddleMoveDataToDevice: | |||||
paddle_list = [paddle.rand((6, 4, 2)) for i in range(10)] | paddle_list = [paddle.rand((6, 4, 2)) for i in range(10)] | ||||
paddle_tuple = tuple(paddle_list) | paddle_tuple = tuple(paddle_list) | ||||
res = paddle_move_data_to_device(paddle_tuple, device="cpu", data_device="gpu:1") | |||||
res = paddle_move_data_to_device(paddle_tuple, device="cpu") | |||||
assert isinstance(res, tuple) | assert isinstance(res, tuple) | ||||
for r in res: | for r in res: | ||||
self.check_cpu(r) | self.check_cpu(r) | ||||
res = paddle_move_data_to_device(paddle_tuple, device="gpu:0", data_device=None) | |||||
res = paddle_move_data_to_device(paddle_tuple, device="gpu:0") | |||||
assert isinstance(res, tuple) | assert isinstance(res, tuple) | ||||
for r in res: | for r in res: | ||||
self.check_gpu(r, 0) | self.check_gpu(r, 0) | ||||
@@ -150,7 +144,7 @@ class TestPaddleMoveDataToDevice: | |||||
"string": "test string" | "string": "test string" | ||||
} | } | ||||
res = paddle_move_data_to_device(paddle_dict, device="gpu:0", data_device=None) | |||||
res = paddle_move_data_to_device(paddle_dict, device="gpu:0") | |||||
assert isinstance(res, dict) | assert isinstance(res, dict) | ||||
self.check_gpu(res["tensor"], 0) | self.check_gpu(res["tensor"], 0) | ||||
assert isinstance(res["list"], list) | assert isinstance(res["list"], list) | ||||
@@ -164,7 +158,7 @@ class TestPaddleMoveDataToDevice: | |||||
self.check_gpu(t, 0) | self.check_gpu(t, 0) | ||||
self.check_gpu(res["dict"]["tensor"], 0) | self.check_gpu(res["dict"]["tensor"], 0) | ||||
res = paddle_move_data_to_device(paddle_dict, device="gpu:0", data_device="cpu") | |||||
res = paddle_move_data_to_device(paddle_dict, device="gpu:0") | |||||
assert isinstance(res, dict) | assert isinstance(res, dict) | ||||
self.check_gpu(res["tensor"], 0) | self.check_gpu(res["tensor"], 0) | ||||
assert isinstance(res["list"], list) | assert isinstance(res["list"], list) | ||||
@@ -178,7 +172,7 @@ class TestPaddleMoveDataToDevice: | |||||
self.check_gpu(t, 0) | self.check_gpu(t, 0) | ||||
self.check_gpu(res["dict"]["tensor"], 0) | self.check_gpu(res["dict"]["tensor"], 0) | ||||
res = paddle_move_data_to_device(paddle_dict, device="cpu", data_device="gpu:0") | |||||
res = paddle_move_data_to_device(paddle_dict, device="cpu") | |||||
assert isinstance(res, dict) | assert isinstance(res, dict) | ||||
self.check_cpu(res["tensor"]) | self.check_cpu(res["tensor"]) | ||||
assert isinstance(res["list"], list) | assert isinstance(res["list"], list) | ||||
@@ -56,13 +56,13 @@ class TestPaddle2Torch: | |||||
res = paddle2torch(paddle_tensor) | res = paddle2torch(paddle_tensor) | ||||
self.check_torch_tensor(res, "cpu", not paddle_tensor.stop_gradient) | self.check_torch_tensor(res, "cpu", not paddle_tensor.stop_gradient) | ||||
res = paddle2torch(paddle_tensor, target_device="cuda:2", no_gradient=None) | |||||
res = paddle2torch(paddle_tensor, device="cuda:2", no_gradient=None) | |||||
self.check_torch_tensor(res, "cuda:2", not paddle_tensor.stop_gradient) | self.check_torch_tensor(res, "cuda:2", not paddle_tensor.stop_gradient) | ||||
res = paddle2torch(paddle_tensor, target_device="cuda:1", no_gradient=True) | |||||
res = paddle2torch(paddle_tensor, device="cuda:1", no_gradient=True) | |||||
self.check_torch_tensor(res, "cuda:1", False) | self.check_torch_tensor(res, "cuda:1", False) | ||||
res = paddle2torch(paddle_tensor, target_device="cuda:1", no_gradient=False) | |||||
res = paddle2torch(paddle_tensor, device="cuda:1", no_gradient=False) | |||||
self.check_torch_tensor(res, "cuda:1", True) | self.check_torch_tensor(res, "cuda:1", True) | ||||
def test_list_transfer(self): | def test_list_transfer(self): | ||||
@@ -76,7 +76,7 @@ class TestPaddle2Torch: | |||||
for t in res: | for t in res: | ||||
self.check_torch_tensor(t, "cuda:1", False) | self.check_torch_tensor(t, "cuda:1", False) | ||||
res = paddle2torch(paddle_list, target_device="cpu", no_gradient=False) | |||||
res = paddle2torch(paddle_list, device="cpu", no_gradient=False) | |||||
assert isinstance(res, list) | assert isinstance(res, list) | ||||
for t in res: | for t in res: | ||||
self.check_torch_tensor(t, "cpu", True) | self.check_torch_tensor(t, "cpu", True) | ||||
@@ -176,13 +176,13 @@ class TestTorch2Paddle: | |||||
res = torch2paddle(torch_tensor) | res = torch2paddle(torch_tensor) | ||||
self.check_paddle_tensor(res, "cpu", True) | self.check_paddle_tensor(res, "cpu", True) | ||||
res = torch2paddle(torch_tensor, target_device="gpu:2", no_gradient=None) | |||||
res = torch2paddle(torch_tensor, device="gpu:2", no_gradient=None) | |||||
self.check_paddle_tensor(res, "gpu:2", True) | self.check_paddle_tensor(res, "gpu:2", True) | ||||
res = torch2paddle(torch_tensor, target_device="gpu:2", no_gradient=True) | |||||
res = torch2paddle(torch_tensor, device="gpu:2", no_gradient=True) | |||||
self.check_paddle_tensor(res, "gpu:2", True) | self.check_paddle_tensor(res, "gpu:2", True) | ||||
res = torch2paddle(torch_tensor, target_device="gpu:2", no_gradient=False) | |||||
res = torch2paddle(torch_tensor, device="gpu:2", no_gradient=False) | |||||
self.check_paddle_tensor(res, "gpu:2", False) | self.check_paddle_tensor(res, "gpu:2", False) | ||||
def test_tensor_list_transfer(self): | def test_tensor_list_transfer(self): | ||||
@@ -196,7 +196,7 @@ class TestTorch2Paddle: | |||||
for t in res: | for t in res: | ||||
self.check_paddle_tensor(t, "cpu", True) | self.check_paddle_tensor(t, "cpu", True) | ||||
res = torch2paddle(torch_list, target_device="gpu:1", no_gradient=False) | |||||
res = torch2paddle(torch_list, device="gpu:1", no_gradient=False) | |||||
assert isinstance(res, list) | assert isinstance(res, list) | ||||
for t in res: | for t in res: | ||||
self.check_paddle_tensor(t, "gpu:1", False) | self.check_paddle_tensor(t, "gpu:1", False) | ||||
@@ -208,7 +208,7 @@ class TestTorch2Paddle: | |||||
torch_list = [torch.rand(6, 4, 2) for i in range(10)] | torch_list = [torch.rand(6, 4, 2) for i in range(10)] | ||||
torch_tuple = tuple(torch_list) | torch_tuple = tuple(torch_list) | ||||
res = torch2paddle(torch_tuple, target_device="cpu") | |||||
res = torch2paddle(torch_tuple, device="cpu") | |||||
assert isinstance(res, tuple) | assert isinstance(res, tuple) | ||||
for t in res: | for t in res: | ||||
self.check_paddle_tensor(t, "cpu", True) | self.check_paddle_tensor(t, "cpu", True) | ||||
@@ -249,6 +249,7 @@ class TestTorch2Paddle: | |||||
# | # | ||||
############################################################################ | ############################################################################ | ||||
@pytest.mark.torchjittor | |||||
class TestJittor2Torch: | class TestJittor2Torch: | ||||
def check_torch_tensor(self, tensor, device, requires_grad): | def check_torch_tensor(self, tensor, device, requires_grad): | ||||
@@ -272,13 +273,13 @@ class TestJittor2Torch: | |||||
res = jittor2torch(jittor_var) | res = jittor2torch(jittor_var) | ||||
self.check_torch_tensor(res, "cpu", True) | self.check_torch_tensor(res, "cpu", True) | ||||
res = jittor2torch(jittor_var, target_device="cuda:2", no_gradient=None) | |||||
res = jittor2torch(jittor_var, device="cuda:2", no_gradient=None) | |||||
self.check_torch_tensor(res, "cuda:2", True) | self.check_torch_tensor(res, "cuda:2", True) | ||||
res = jittor2torch(jittor_var, target_device="cuda:2", no_gradient=True) | |||||
res = jittor2torch(jittor_var, device="cuda:2", no_gradient=True) | |||||
self.check_torch_tensor(res, "cuda:2", False) | self.check_torch_tensor(res, "cuda:2", False) | ||||
res = jittor2torch(jittor_var, target_device="cuda:2", no_gradient=False) | |||||
res = jittor2torch(jittor_var, device="cuda:2", no_gradient=False) | |||||
self.check_torch_tensor(res, "cuda:2", True) | self.check_torch_tensor(res, "cuda:2", True) | ||||
def test_var_list_transfer(self): | def test_var_list_transfer(self): | ||||
@@ -292,7 +293,7 @@ class TestJittor2Torch: | |||||
for t in res: | for t in res: | ||||
self.check_torch_tensor(t, "cpu", True) | self.check_torch_tensor(t, "cpu", True) | ||||
res = jittor2torch(jittor_list, target_device="cuda:1", no_gradient=False) | |||||
res = jittor2torch(jittor_list, device="cuda:1", no_gradient=False) | |||||
assert isinstance(res, list) | assert isinstance(res, list) | ||||
for t in res: | for t in res: | ||||
self.check_torch_tensor(t, "cuda:1", True) | self.check_torch_tensor(t, "cuda:1", True) | ||||
@@ -304,7 +305,7 @@ class TestJittor2Torch: | |||||
jittor_list = [jittor.rand((6, 4, 2)) for i in range(10)] | jittor_list = [jittor.rand((6, 4, 2)) for i in range(10)] | ||||
jittor_tuple = tuple(jittor_list) | jittor_tuple = tuple(jittor_list) | ||||
res = jittor2torch(jittor_tuple, target_device="cpu") | |||||
res = jittor2torch(jittor_tuple, device="cpu") | |||||
assert isinstance(res, tuple) | assert isinstance(res, tuple) | ||||
for t in res: | for t in res: | ||||
self.check_torch_tensor(t, "cpu", True) | self.check_torch_tensor(t, "cpu", True) | ||||
@@ -345,6 +346,7 @@ class TestJittor2Torch: | |||||
# | # | ||||
############################################################################ | ############################################################################ | ||||
@pytest.mark.torchjittor | |||||
class TestTorch2Jittor: | class TestTorch2Jittor: | ||||
def check_jittor_var(self, var, requires_grad): | def check_jittor_var(self, var, requires_grad): | ||||
@@ -4,4 +4,5 @@ markers = | |||||
paddle | paddle | ||||
paddledist | paddledist | ||||
jittor | jittor | ||||
torchpaddle | |||||
torchpaddle | |||||
torchjittor |