"git@developer.sourcefind.cn:norm/vllm.git" did not exist on "ef978fe4111b0eb91c81eceba4d9791b94c7ffbf"
Unverified Commit 734ee253 authored by Philip Meier's avatar Philip Meier Committed by GitHub
Browse files

fix test_data_loader on Windows and macOS (#5912)



* fix test_data_loader on Windows and macOS

* Update test/test_prototype_builtin_datasets.py
Co-authored-by: default avatarNicolas Hug <contact@nicolas-hug.com>
Co-authored-by: default avatarNicolas Hug <contact@nicolas-hug.com>
parent a945ec63
...@@ -125,6 +125,12 @@ class TestCommon: ...@@ -125,6 +125,12 @@ class TestCommon:
pickle.dumps(dataset) pickle.dumps(dataset)
# This has to be a proper function, since lambda's or local functions
# cannot be pickled, but this is a requirement for the DataLoader with
# multiprocessing, i.e. num_workers > 0
def _collate_fn(self, batch):
return batch
@pytest.mark.parametrize("num_workers", [0, 1]) @pytest.mark.parametrize("num_workers", [0, 1])
@parametrize_dataset_mocks(DATASET_MOCKS) @parametrize_dataset_mocks(DATASET_MOCKS)
def test_data_loader(self, test_home, dataset_mock, config, num_workers): def test_data_loader(self, test_home, dataset_mock, config, num_workers):
...@@ -135,7 +141,7 @@ class TestCommon: ...@@ -135,7 +141,7 @@ class TestCommon:
dataset, dataset,
batch_size=2, batch_size=2,
num_workers=num_workers, num_workers=num_workers,
collate_fn=lambda batch: batch, collate_fn=self._collate_fn,
) )
next(iter(dl)) next(iter(dl))
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment