Unverified Commit 2ba586d5 authored by Nicolas Hug's avatar Nicolas Hug Committed by GitHub
Browse files

Document that datasets support pathlib.Path (#8321)


Co-authored-by: default avatarPhilip Meier <github.pmeier@posteo.de>
parent 03251754
from pathlib import Path
from typing import Any, Callable, Optional, Tuple
from typing import Any, Callable, Optional, Tuple, Union
from PIL import Image
......@@ -12,7 +12,7 @@ class Imagenette(VisionDataset):
"""`Imagenette <https://github.com/fastai/imagenette#imagenette-1>`_ image classification dataset.
Args:
root (string): Root directory of the Imagenette dataset.
root (str or ``pathlib.Path``): Root directory of the Imagenette dataset.
split (string, optional): The dataset split. Supports ``"train"`` (default), and ``"val"``.
size (string, optional): The image size. Supports ``"full"`` (default), ``"320px"``, and ``"160px"``.
download (bool, optional): If ``True``, downloads the dataset components and places them in ``root``. Already
......@@ -48,7 +48,7 @@ class Imagenette(VisionDataset):
def __init__(
self,
root: str,
root: Union[str, Path],
split: str = "train",
size: str = "full",
download=False,
......
import os
import os.path
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
from PIL import Image
......@@ -32,7 +33,7 @@ class INaturalist(VisionDataset):
"""`iNaturalist <https://github.com/visipedia/inat_comp>`_ Dataset.
Args:
root (string): Root directory of dataset where the image files are stored.
root (str or ``pathlib.Path``): Root directory of dataset where the image files are stored.
This class does not require/use annotation files.
version (string, optional): Which version of the dataset to download/use. One of
'2017', '2018', '2019', '2021_train', '2021_train_mini', '2021_valid'.
......@@ -65,7 +66,7 @@ class INaturalist(VisionDataset):
def __init__(
self,
root: str,
root: Union[str, Path],
version: str = "2021_train",
target_type: Union[List[str], str] = "full",
transform: Optional[Callable] = None,
......
......@@ -5,7 +5,8 @@ import urllib
from functools import partial
from multiprocessing import Pool
from os import path
from typing import Any, Callable, Dict, Optional, Tuple
from pathlib import Path
from typing import Any, Callable, Dict, Optional, Tuple, Union
from torch import Tensor
......@@ -35,7 +36,7 @@ class Kinetics(VisionDataset):
frames in a video might be present.
Args:
root (string): Root directory of the Kinetics Dataset.
root (str or ``pathlib.Path``): Root directory of the Kinetics Dataset.
Directory should be structured as follows:
.. code::
......@@ -90,7 +91,7 @@ class Kinetics(VisionDataset):
def __init__(
self,
root: str,
root: Union[str, Path],
frames_per_clip: int,
num_classes: str = "400",
split: str = "train",
......
import csv
import os
from typing import Any, Callable, List, Optional, Tuple
from pathlib import Path
from typing import Any, Callable, List, Optional, Tuple, Union
from PIL import Image
......@@ -14,7 +15,7 @@ class Kitti(VisionDataset):
It corresponds to the "left color images of object" dataset, for object detection.
Args:
root (string): Root directory where images are downloaded to.
root (str or ``pathlib.Path``): Root directory where images are downloaded to.
Expects the following folder structure if download=False:
.. code::
......@@ -51,7 +52,7 @@ class Kitti(VisionDataset):
def __init__(
self,
root: str,
root: Union[str, Path],
train: bool = True,
transform: Optional[Callable] = None,
target_transform: Optional[Callable] = None,
......
import os
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
from PIL import Image
......@@ -31,7 +32,7 @@ class _LFW(VisionDataset):
def __init__(
self,
root: str,
root: Union[str, Path],
split: str,
image_set: str,
view: str,
......@@ -95,7 +96,7 @@ class LFWPeople(_LFW):
"""`LFW <http://vis-www.cs.umass.edu/lfw/>`_ Dataset.
Args:
root (string): Root directory of dataset where directory
root (str or ``pathlib.Path``): Root directory of dataset where directory
``lfw-py`` exists or will be saved to if download is set to True.
split (string, optional): The image split to use. Can be one of ``train``, ``test``,
``10fold`` (default).
......@@ -177,7 +178,7 @@ class LFWPairs(_LFW):
"""`LFW <http://vis-www.cs.umass.edu/lfw/>`_ Dataset.
Args:
root (string): Root directory of dataset where directory
root (str or ``pathlib.Path``): Root directory of dataset where directory
``lfw-py`` exists or will be saved to if download is set to True.
split (string, optional): The image split to use. Can be one of ``train``, ``test``,
``10fold``. Defaults to ``10fold``.
......
......@@ -3,6 +3,7 @@ import os.path
import pickle
import string
from collections.abc import Iterable
from pathlib import Path
from typing import Any, Callable, cast, List, Optional, Tuple, Union
from PIL import Image
......@@ -60,7 +61,7 @@ class LSUN(VisionDataset):
``pip install lmdb``
Args:
root (string): Root directory for the database files.
root (str or ``pathlib.Path``): Root directory for the database files.
classes (string or list): One of {'train', 'val', 'test'} or a list of
categories to load. e,g. ['bedroom_train', 'church_outdoor_train'].
transform (callable, optional): A function/transform that takes in a PIL image
......@@ -71,7 +72,7 @@ class LSUN(VisionDataset):
def __init__(
self,
root: str,
root: Union[str, Path],
classes: Union[str, List[str]] = "train",
transform: Optional[Callable] = None,
target_transform: Optional[Callable] = None,
......
......@@ -5,7 +5,8 @@ import shutil
import string
import sys
import warnings
from typing import Any, Callable, Dict, List, Optional, Tuple
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
from urllib.error import URLError
import numpy as np
......@@ -20,7 +21,7 @@ class MNIST(VisionDataset):
"""`MNIST <http://yann.lecun.com/exdb/mnist/>`_ Dataset.
Args:
root (string): Root directory of dataset where ``MNIST/raw/train-images-idx3-ubyte``
root (str or ``pathlib.Path``): Root directory of dataset where ``MNIST/raw/train-images-idx3-ubyte``
and ``MNIST/raw/t10k-images-idx3-ubyte`` exist.
train (bool, optional): If True, creates dataset from ``train-images-idx3-ubyte``,
otherwise from ``t10k-images-idx3-ubyte``.
......@@ -82,7 +83,7 @@ class MNIST(VisionDataset):
def __init__(
self,
root: str,
root: Union[str, Path],
train: bool = True,
transform: Optional[Callable] = None,
target_transform: Optional[Callable] = None,
......@@ -203,7 +204,7 @@ class FashionMNIST(MNIST):
"""`Fashion-MNIST <https://github.com/zalandoresearch/fashion-mnist>`_ Dataset.
Args:
root (string): Root directory of dataset where ``FashionMNIST/raw/train-images-idx3-ubyte``
root (str or ``pathlib.Path``): Root directory of dataset where ``FashionMNIST/raw/train-images-idx3-ubyte``
and ``FashionMNIST/raw/t10k-images-idx3-ubyte`` exist.
train (bool, optional): If True, creates dataset from ``train-images-idx3-ubyte``,
otherwise from ``t10k-images-idx3-ubyte``.
......@@ -231,7 +232,7 @@ class KMNIST(MNIST):
"""`Kuzushiji-MNIST <https://github.com/rois-codh/kmnist>`_ Dataset.
Args:
root (string): Root directory of dataset where ``KMNIST/raw/train-images-idx3-ubyte``
root (str or ``pathlib.Path``): Root directory of dataset where ``KMNIST/raw/train-images-idx3-ubyte``
and ``KMNIST/raw/t10k-images-idx3-ubyte`` exist.
train (bool, optional): If True, creates dataset from ``train-images-idx3-ubyte``,
otherwise from ``t10k-images-idx3-ubyte``.
......@@ -259,7 +260,7 @@ class EMNIST(MNIST):
"""`EMNIST <https://www.westernsydney.edu.au/bens/home/reproducible_research/emnist>`_ Dataset.
Args:
root (string): Root directory of dataset where ``EMNIST/raw/train-images-idx3-ubyte``
root (str or ``pathlib.Path``): Root directory of dataset where ``EMNIST/raw/train-images-idx3-ubyte``
and ``EMNIST/raw/t10k-images-idx3-ubyte`` exist.
split (string): The dataset has 6 different splits: ``byclass``, ``bymerge``,
``balanced``, ``letters``, ``digits`` and ``mnist``. This argument specifies
......@@ -290,7 +291,7 @@ class EMNIST(MNIST):
"mnist": list(string.digits),
}
def __init__(self, root: str, split: str, **kwargs: Any) -> None:
def __init__(self, root: Union[str, Path], split: str, **kwargs: Any) -> None:
self.split = verify_str_arg(split, "split", self.splits)
self.training_file = self._training_file(split)
self.test_file = self._test_file(split)
......@@ -343,7 +344,7 @@ class QMNIST(MNIST):
"""`QMNIST <https://github.com/facebookresearch/qmnist>`_ Dataset.
Args:
root (string): Root directory of dataset whose ``raw``
root (str or ``pathlib.Path``): Root directory of dataset whose ``raw``
subdir contains binary files of the datasets.
what (string,optional): Can be 'train', 'test', 'test10k',
'test50k', or 'nist' for respectively the mnist compatible
......@@ -416,7 +417,7 @@ class QMNIST(MNIST):
]
def __init__(
self, root: str, what: Optional[str] = None, compat: bool = True, train: bool = True, **kwargs: Any
self, root: Union[str, Path], what: Optional[str] = None, compat: bool = True, train: bool = True, **kwargs: Any
) -> None:
if what is None:
what = "train" if train else "test"
......
import os.path
from typing import Callable, Optional
from pathlib import Path
from typing import Callable, Optional, Union
import numpy as np
import torch
......@@ -11,7 +12,7 @@ class MovingMNIST(VisionDataset):
"""`MovingMNIST <http://www.cs.toronto.edu/~nitish/unsupervised_video/>`_ Dataset.
Args:
root (string): Root directory of dataset where ``MovingMNIST/mnist_test_seq.npy`` exists.
root (str or ``pathlib.Path``): Root directory of dataset where ``MovingMNIST/mnist_test_seq.npy`` exists.
split (string, optional): The dataset split, supports ``None`` (default), ``"train"`` and ``"test"``.
If ``split=None``, the full data is returned.
split_ratio (int, optional): The split ratio of number of frames. If ``split="train"``, the first split
......@@ -28,7 +29,7 @@ class MovingMNIST(VisionDataset):
def __init__(
self,
root: str,
root: Union[str, Path],
split: Optional[str] = None,
split_ratio: int = 10,
download: bool = False,
......
from os.path import join
from typing import Any, Callable, List, Optional, Tuple
from pathlib import Path
from typing import Any, Callable, List, Optional, Tuple, Union
from PIL import Image
......@@ -11,7 +12,7 @@ class Omniglot(VisionDataset):
"""`Omniglot <https://github.com/brendenlake/omniglot>`_ Dataset.
Args:
root (string): Root directory of dataset where directory
root (str or ``pathlib.Path``): Root directory of dataset where directory
``omniglot-py`` exists.
background (bool, optional): If True, creates dataset from the "background" set, otherwise
creates from the "evaluation" set. This terminology is defined by the authors.
......@@ -33,7 +34,7 @@ class Omniglot(VisionDataset):
def __init__(
self,
root: str,
root: Union[str, Path],
background: bool = True,
transform: Optional[Callable] = None,
target_transform: Optional[Callable] = None,
......
......@@ -13,7 +13,7 @@ class OxfordIIITPet(VisionDataset):
"""`Oxford-IIIT Pet Dataset <https://www.robots.ox.ac.uk/~vgg/data/pets/>`_.
Args:
root (string): Root directory of the dataset.
root (str or ``pathlib.Path``): Root directory of the dataset.
split (string, optional): The dataset split, supports ``"trainval"`` (default) or ``"test"``.
target_types (string, sequence of strings, optional): Types of target to use. Can be ``category`` (default) or
``segmentation``. Can also be a list to output a tuple with all specified target types. The types represent:
......@@ -38,7 +38,7 @@ class OxfordIIITPet(VisionDataset):
def __init__(
self,
root: str,
root: Union[str, pathlib.Path],
split: str = "trainval",
target_types: Union[Sequence[str], str] = "category",
transforms: Optional[Callable] = None,
......
import pathlib
from typing import Any, Callable, Optional, Tuple
from typing import Any, Callable, Optional, Tuple, Union
from PIL import Image
......@@ -18,7 +18,7 @@ class PCAM(VisionDataset):
This dataset requires the ``h5py`` package which you can install with ``pip install h5py``.
Args:
root (string): Root directory of the dataset.
root (str or ``pathlib.Path``): Root directory of the dataset.
split (string, optional): The dataset split, supports ``"train"`` (default), ``"test"`` or ``"val"``.
transform (callable, optional): A function/transform that takes in a PIL image and returns a transformed
version. E.g, ``transforms.RandomCrop``.
......@@ -72,7 +72,7 @@ class PCAM(VisionDataset):
def __init__(
self,
root: str,
root: Union[str, pathlib.Path],
split: str = "train",
transform: Optional[Callable] = None,
target_transform: Optional[Callable] = None,
......
import os
from pathlib import Path
from typing import Any, Callable, List, Optional, Tuple, Union
import numpy as np
......@@ -24,7 +25,7 @@ class PhotoTour(VisionDataset):
Args:
root (string): Root directory where images are.
root (str or ``pathlib.Path``): Root directory where images are.
name (string): Name of the dataset to load.
transform (callable, optional): A function/transform that takes in a PIL image
and returns a transformed version.
......@@ -87,7 +88,12 @@ class PhotoTour(VisionDataset):
matches_files = "m50_100000_100000_0.txt"
def __init__(
self, root: str, name: str, train: bool = True, transform: Optional[Callable] = None, download: bool = False
self,
root: Union[str, Path],
name: str,
train: bool = True,
transform: Optional[Callable] = None,
download: bool = False,
) -> None:
super().__init__(root, transform=transform)
self.name = name
......
import os
from os import path
from typing import Any, Callable, Dict, List, Optional, Tuple
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
from urllib.parse import urljoin
from .folder import default_loader
......@@ -12,7 +13,7 @@ class Places365(VisionDataset):
r"""`Places365 <http://places2.csail.mit.edu/index.html>`_ classification dataset.
Args:
root (string): Root directory of the Places365 dataset.
root (str or ``pathlib.Path``): Root directory of the Places365 dataset.
split (string, optional): The dataset split. Can be one of ``train-standard`` (default), ``train-challenge``,
``val``.
small (bool, optional): If ``True``, uses the small images, i.e. resized to 256 x 256 pixels, instead of the
......@@ -62,7 +63,7 @@ class Places365(VisionDataset):
def __init__(
self,
root: str,
root: Union[str, Path],
split: str = "train-standard",
small: bool = False,
download: bool = False,
......
from pathlib import Path
from typing import Any, Callable, Optional, Tuple
from typing import Any, Callable, Optional, Tuple, Union
import PIL.Image
......@@ -20,7 +20,7 @@ class RenderedSST2(VisionDataset):
(444 positive and 428 negative), and a test split containing 1821 images (909 positive and 912 negative).
Args:
root (string): Root directory of the dataset.
root (str or ``pathlib.Path``): Root directory of the dataset.
split (string, optional): The dataset split, supports ``"train"`` (default), `"val"` and ``"test"``.
transform (callable, optional): A function/transform that takes in a PIL image and returns a transformed
version. E.g, ``transforms.RandomCrop``.
......@@ -35,7 +35,7 @@ class RenderedSST2(VisionDataset):
def __init__(
self,
root: str,
root: Union[str, Path],
split: str = "train",
transform: Optional[Callable] = None,
target_transform: Optional[Callable] = None,
......
import os
import shutil
from typing import Any, Callable, Optional, Tuple
from pathlib import Path
from typing import Any, Callable, Optional, Tuple, Union
import numpy as np
from PIL import Image
......@@ -27,7 +28,7 @@ class SBDataset(VisionDataset):
This class needs `scipy <https://docs.scipy.org/doc/>`_ to load target files from `.mat` format.
Args:
root (string): Root directory of the Semantic Boundaries Dataset
root (str or ``pathlib.Path``): Root directory of the Semantic Boundaries Dataset
image_set (string, optional): Select the image_set to use, ``train``, ``val`` or ``train_noval``.
Image set ``train_noval`` excludes VOC 2012 val images.
mode (string, optional): Select target type. Possible values 'boundaries' or 'segmentation'.
......@@ -51,7 +52,7 @@ class SBDataset(VisionDataset):
def __init__(
self,
root: str,
root: Union[str, Path],
image_set: str = "train",
mode: str = "boundaries",
download: bool = False,
......
import os
from typing import Any, Callable, Optional, Tuple
from pathlib import Path
from typing import Any, Callable, Optional, Tuple, Union
from PIL import Image
......@@ -11,7 +12,7 @@ class SBU(VisionDataset):
"""`SBU Captioned Photo <http://www.cs.virginia.edu/~vicente/sbucaptions/>`_ Dataset.
Args:
root (string): Root directory of dataset where tarball
root (str or ``pathlib.Path``): Root directory of dataset where tarball
``SBUCaptionedPhotoDataset.tar.gz`` exists.
transform (callable, optional): A function/transform that takes in a PIL image
and returns a transformed version. E.g, ``transforms.RandomCrop``
......@@ -28,7 +29,7 @@ class SBU(VisionDataset):
def __init__(
self,
root: str,
root: Union[str, Path],
transform: Optional[Callable] = None,
target_transform: Optional[Callable] = None,
download: bool = True,
......
import os.path
from typing import Any, Callable, Optional, Tuple
from pathlib import Path
from typing import Any, Callable, Optional, Tuple, Union
import numpy as np
from PIL import Image
......@@ -12,7 +13,7 @@ class SEMEION(VisionDataset):
r"""`SEMEION <http://archive.ics.uci.edu/ml/datasets/semeion+handwritten+digit>`_ Dataset.
Args:
root (string): Root directory of dataset where directory
root (str or ``pathlib.Path``): Root directory of dataset where directory
``semeion.py`` exists.
transform (callable, optional): A function/transform that takes in a PIL image
and returns a transformed version. E.g, ``transforms.RandomCrop``
......@@ -29,7 +30,7 @@ class SEMEION(VisionDataset):
def __init__(
self,
root: str,
root: Union[str, Path],
transform: Optional[Callable] = None,
target_transform: Optional[Callable] = None,
download: bool = True,
......
import pathlib
from typing import Any, Callable, Optional, Tuple
from typing import Any, Callable, Optional, Tuple, Union
from PIL import Image
......@@ -21,7 +21,7 @@ class StanfordCars(VisionDataset):
This class needs `scipy <https://docs.scipy.org/doc/>`_ to load target files from `.mat` format.
Args:
root (string): Root directory of dataset
root (str or ``pathlib.Path``): Root directory of dataset
split (string, optional): The dataset split, supports ``"train"`` (default) or ``"test"``.
transform (callable, optional): A function/transform that takes in a PIL image
and returns a transformed version. E.g, ``transforms.RandomCrop``
......@@ -35,7 +35,7 @@ class StanfordCars(VisionDataset):
def __init__(
self,
root: str,
root: Union[str, pathlib.Path],
split: str = "train",
transform: Optional[Callable] = None,
target_transform: Optional[Callable] = None,
......
import os.path
from typing import Any, Callable, cast, Optional, Tuple
from pathlib import Path
from typing import Any, Callable, cast, Optional, Tuple, Union
import numpy as np
from PIL import Image
......@@ -12,7 +13,7 @@ class STL10(VisionDataset):
"""`STL10 <https://cs.stanford.edu/~acoates/stl10/>`_ Dataset.
Args:
root (string): Root directory of dataset where directory
root (str or ``pathlib.Path``): Root directory of dataset where directory
``stl10_binary`` exists.
split (string): One of {'train', 'test', 'unlabeled', 'train+unlabeled'}.
Accordingly, dataset is selected.
......@@ -45,7 +46,7 @@ class STL10(VisionDataset):
def __init__(
self,
root: str,
root: Union[str, Path],
split: str = "train",
folds: Optional[int] = None,
transform: Optional[Callable] = None,
......
from pathlib import Path
from typing import Any, Callable, Optional, Tuple
from typing import Any, Callable, Optional, Tuple, Union
import PIL.Image
......@@ -14,7 +14,7 @@ class SUN397(VisionDataset):
397 categories with 108'754 images.
Args:
root (string): Root directory of the dataset.
root (str or ``pathlib.Path``): Root directory of the dataset.
transform (callable, optional): A function/transform that takes in a PIL image and returns a transformed
version. E.g, ``transforms.RandomCrop``.
target_transform (callable, optional): A function/transform that takes in the target and transforms it.
......@@ -28,7 +28,7 @@ class SUN397(VisionDataset):
def __init__(
self,
root: str,
root: Union[str, Path],
transform: Optional[Callable] = None,
target_transform: Optional[Callable] = None,
download: bool = False,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment