Unverified Commit 578c1546 authored by Philip Meier's avatar Philip Meier Committed by GitHub
Browse files

*FileLoader -> *FileOpener (#5128)



* *FileLoader -> *FileOpener

* fix opening mode
Co-authored-by: default avatarKevin Tse <NivekT@users.noreply.github.com>
parent 93ec8bfd
......@@ -7,8 +7,7 @@ from typing import Callable, Optional, Collection
from typing import Union, Tuple, List, Dict, Any
import torch
from torch.utils.data import IterDataPipe
from torch.utils.data.datapipes.iter import FileLister, FileLoader, Mapper, Shuffler, Filter
from torchdata.datapipes.iter import IterDataPipe, FileLister, FileOpener, Mapper, Shuffler, Filter
from torchvision.prototype.datasets.decoder import pil
from torchvision.prototype.datasets.utils._internal import INFINITE_BUFFER_SIZE, hint_sharding
......@@ -54,7 +53,7 @@ def from_data_folder(
dp: IterDataPipe = Filter(dp, functools.partial(_is_not_top_level_file, root=root))
dp = hint_sharding(dp)
dp = Shuffler(dp, buffer_size=INFINITE_BUFFER_SIZE)
dp = FileLoader(dp)
dp = FileOpener(dp, mode="rb")
return (
Mapper(dp, functools.partial(_collate_and_decode_data, root=root, categories=categories, decoder=decoder)),
categories,
......
......@@ -30,7 +30,7 @@ import PIL.Image
import torch
import torch.distributed as dist
import torch.utils.data
from torchdata.datapipes.iter import IoPathFileLister, IoPathFileLoader, IterDataPipe, ShardingFilter, Shuffler
from torchdata.datapipes.iter import IoPathFileLister, IoPathFileOpener, IterDataPipe, ShardingFilter, Shuffler
from torchdata.datapipes.utils import StreamWrapper
......@@ -254,7 +254,7 @@ def _make_sharded_datapipe(root: str, dataset_size: int) -> IterDataPipe[Dict[st
dp = IoPathFileLister(root=root)
dp = SharderDataPipe(dp)
dp = dp.shuffle(buffer_size=INFINITE_BUFFER_SIZE)
dp = IoPathFileLoader(dp, mode="rb")
dp = IoPathFileOpener(dp, mode="rb")
dp = PicklerDataPipe(dp)
# dp = dp.cycle(2)
dp = TakerDataPipe(dp, dataset_size)
......
......@@ -9,7 +9,7 @@ from urllib.parse import urlparse
from torchdata.datapipes.iter import (
IterableWrapper,
FileLister,
FileLoader,
FileOpener,
IterDataPipe,
ZipArchiveReader,
TarArchiveReader,
......@@ -62,9 +62,9 @@ class OnlineResource(abc.ABC):
def _default_loader(self, path: pathlib.Path) -> IterDataPipe[Tuple[str, IO]]:
if path.is_dir():
return FileLoader(FileLister(str(path), recursive=True))
return FileOpener(FileLister(str(path), recursive=True), mode="rb")
dp = FileLoader(IterableWrapper((str(path),)))
dp = FileOpener(IterableWrapper((str(path),)), mode="rb")
archive_loader = self._guess_archive_loader(path)
if archive_loader:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment