From 38e578f6b58447a0c948b515209f6597cd1284ef Mon Sep 17 00:00:00 2001 From: x54-729 <17307130121@fudan.edu.cn> Date: Tue, 3 May 2022 15:05:30 +0000 Subject: [PATCH] =?UTF-8?q?PaddleDataLoader=201=E3=80=81=E5=88=9D=E5=A7=8B?= =?UTF-8?q?=E5=8C=96=E6=97=B6=E5=A6=82=E6=9E=9C=20batch=5Fsampler=20?= =?UTF-8?q?=E4=B8=BA=20None=20=E6=97=B6=E9=87=8D=E7=BD=AE=20shuffle=20?= =?UTF-8?q?=E7=AD=89=E5=8F=82=E6=95=B0=202=E3=80=81=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E4=BE=8B=E4=B8=AD=E5=88=A0=E9=99=A4=E4=BA=86=E4=B8=8D=E5=BF=85?= =?UTF-8?q?=E8=A6=81=E7=9A=84=20test=5Fv2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fastNLP/core/dataloaders/paddle_dataloader/fdl.py | 6 +++++- tests/core/dataloaders/paddle_dataloader/test_fdl.py | 8 -------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/fastNLP/core/dataloaders/paddle_dataloader/fdl.py b/fastNLP/core/dataloaders/paddle_dataloader/fdl.py index fa99be22..952759f7 100644 --- a/fastNLP/core/dataloaders/paddle_dataloader/fdl.py +++ b/fastNLP/core/dataloaders/paddle_dataloader/fdl.py @@ -8,11 +8,12 @@ from typing import Callable, List, Optional, Union, Dict, Sequence from fastNLP.envs.imports import _NEED_IMPORT_PADDLE if _NEED_IMPORT_PADDLE: - from paddle.io import DataLoader, Dataset + from paddle.io import DataLoader, Dataset, Sampler from paddle.fluid.dataloader.collate import default_collate_fn else: from fastNLP.core.utils.dummy_class import DummyClass as Dataset from fastNLP.core.utils.dummy_class import DummyClass as DataLoader + from fastNLP.core.utils.dummy_class import DummyClass as Sampler from fastNLP.core.collators.collator import Collator from fastNLP.core.dataloaders.utils import indice_collate_wrapper @@ -58,6 +59,9 @@ class PaddleDataLoader(DataLoader): if batch_sampler is None: batch_sampler = RandomBatchSampler(dataset, batch_size=batch_size, shuffle=shuffle, drop_last=drop_last) + batch_size = 1 + shuffle = False + drop_last = False super(PaddleDataLoader, self).__init__(dataset=dataset, feed_list=feed_list, places=places, return_list=return_list, batch_sampler=batch_sampler, diff --git a/tests/core/dataloaders/paddle_dataloader/test_fdl.py b/tests/core/dataloaders/paddle_dataloader/test_fdl.py index d8ba521b..6632ad17 100644 --- a/tests/core/dataloaders/paddle_dataloader/test_fdl.py +++ b/tests/core/dataloaders/paddle_dataloader/test_fdl.py @@ -58,11 +58,3 @@ class TestPaddle: for batch in fdl1: assert batch['image'].shape == [4, 10, 5] print(batch) - - def test_v2(self): - from fastNLP.core.collators import Collator - logger.setLevel("DEBUG") - data = [paddle.Tensor(np.random.random((10, 5)).astype('float32')), paddle.Tensor(np.random.random((10, 5)).astype('float32'))] - col = Collator(backend="jittor") - res = col(data) - print(res) \ No newline at end of file