remove torchvision.prototype module and related tests / CI from release branch (#7983)

torchvision/prototype/datasets/_builtin/coco.py

-import pathlib
-import re
-from collections import defaultdict, OrderedDict
-from typing import Any, BinaryIO, cast, Dict, List, Optional, Tuple, Union
-
-import torch
-from torchdata.datapipes.iter import (
-    Demultiplexer,
-    Filter,
-    Grouper,
-    IterDataPipe,
-    IterKeyZipper,
-    JsonParser,
-    Mapper,
-    UnBatcher,
-)
-from torchvision.prototype.datasets.utils import Dataset, EncodedImage, HttpResource, OnlineResource
-from torchvision.prototype.datasets.utils._internal import (
-    getitem,
-    hint_sharding,
-    hint_shuffling,
-    INFINITE_BUFFER_SIZE,
-    MappingIterator,
-    path_accessor,
-    read_categories_file,
-)
-from torchvision.prototype.tv_tensors import Label
-from torchvision.tv_tensors import BoundingBoxes, Mask
-
-from .._api import register_dataset, register_info
-
-
-NAME = "coco"
-
-
-@register_info(NAME)
-def _info() -> Dict[str, Any]:
-    categories, super_categories = zip(*read_categories_file(NAME))
-    return dict(categories=categories, super_categories=super_categories)
-
-
-@register_dataset(NAME)
-class Coco(Dataset):
-    """
-    - **homepage**: https://cocodataset.org/
-    - **dependencies**:
-        - <pycocotools `https://github.com/cocodataset/cocoapi`>_
-    """
-
-    def __init__(
-        self,
-        root: Union[str, pathlib.Path],
-        *,
-        split: str = "train",
-        year: str = "2017",
-        annotations: Optional[str] = "instances",
-        skip_integrity_check: bool = False,
-    ) -> None:
-        self._split = self._verify_str_arg(split, "split", {"train", "val"})
-        self._year = self._verify_str_arg(year, "year", {"2017", "2014"})
-        self._annotations = (
-            self._verify_str_arg(annotations, "annotations", self._ANN_DECODERS.keys())
-            if annotations is not None
-            else None
-        )
-
-        info = _info()
-        categories, super_categories = info["categories"], info["super_categories"]
-        self._categories = categories
-        self._category_to_super_category = dict(zip(categories, super_categories))
-
-        super().__init__(root, dependencies=("pycocotools",), skip_integrity_check=skip_integrity_check)
-
-    _IMAGE_URL_BASE = "http://images.cocodataset.org/zips"
-
-    _IMAGES_CHECKSUMS = {
-        ("2014", "train"): "ede4087e640bddba550e090eae701092534b554b42b05ac33f0300b984b31775",
-        ("2014", "val"): "fe9be816052049c34717e077d9e34aa60814a55679f804cd043e3cbee3b9fde0",
-        ("2017", "train"): "69a8bb58ea5f8f99d24875f21416de2e9ded3178e903f1f7603e283b9e06d929",
-        ("2017", "val"): "4f7e2ccb2866ec5041993c9cf2a952bbed69647b115d0f74da7ce8f4bef82f05",
-    }
-
-    _META_URL_BASE = "http://images.cocodataset.org/annotations"
-
-    _META_CHECKSUMS = {
-        "2014": "031296bbc80c45a1d1f76bf9a90ead27e94e99ec629208449507a4917a3bf009",
-        "2017": "113a836d90195ee1f884e704da6304dfaaecff1f023f49b6ca93c4aaae470268",
-    }
-
-    def _resources(self) -> List[OnlineResource]:
-        images = HttpResource(
-            f"{self._IMAGE_URL_BASE}/{self._split}{self._year}.zip",
-            sha256=self._IMAGES_CHECKSUMS[(self._year, self._split)],
-        )
-        meta = HttpResource(
-            f"{self._META_URL_BASE}/annotations_trainval{self._year}.zip",
-            sha256=self._META_CHECKSUMS[self._year],
-        )
-        return [images, meta]
-
-    def _segmentation_to_mask(
-        self, segmentation: Any, *, is_crowd: bool, spatial_size: Tuple[int, int]
-    ) -> torch.Tensor:
-        from pycocotools import mask
-
-        if is_crowd:
-            segmentation = mask.frPyObjects(segmentation, *spatial_size)
-        else:
-            segmentation = mask.merge(mask.frPyObjects(segmentation, *spatial_size))
-
-        return torch.from_numpy(mask.decode(segmentation)).to(torch.bool)
-
-    def _decode_instances_anns(self, anns: List[Dict[str, Any]], image_meta: Dict[str, Any]) -> Dict[str, Any]:
-        spatial_size = (image_meta["height"], image_meta["width"])
-        labels = [ann["category_id"] for ann in anns]
-        return dict(
-            segmentations=Mask(
-                torch.stack(
-                    [
-                        self._segmentation_to_mask(
-                            ann["segmentation"], is_crowd=ann["iscrowd"], spatial_size=spatial_size
-                        )
-                        for ann in anns
-                    ]
-                )
-            ),
-            areas=torch.as_tensor([ann["area"] for ann in anns]),
-            crowds=torch.as_tensor([ann["iscrowd"] for ann in anns], dtype=torch.bool),
-            bounding_boxes=BoundingBoxes(
-                [ann["bbox"] for ann in anns],
-                format="xywh",
-                spatial_size=spatial_size,
-            ),
-            labels=Label(labels, categories=self._categories),
-            super_categories=[self._category_to_super_category[self._categories[label]] for label in labels],
-            ann_ids=[ann["id"] for ann in anns],
-        )
-
-    def _decode_captions_ann(self, anns: List[Dict[str, Any]], image_meta: Dict[str, Any]) -> Dict[str, Any]:
-        return dict(
-            captions=[ann["caption"] for ann in anns],
-            ann_ids=[ann["id"] for ann in anns],
-        )
-
-    _ANN_DECODERS = OrderedDict(
-        [
-            ("instances", _decode_instances_anns),
-            ("captions", _decode_captions_ann),
-        ]
-    )
-
-    _META_FILE_PATTERN = re.compile(
-        rf"(?P<annotations>({'|'.join(_ANN_DECODERS.keys())}))_(?P<split>[a-zA-Z]+)(?P<year>\d+)[.]json"
-    )
-
-    def _filter_meta_files(self, data: Tuple[str, Any]) -> bool:
-        match = self._META_FILE_PATTERN.match(pathlib.Path(data[0]).name)
-        return bool(
-            match
-            and match["split"] == self._split
-            and match["year"] == self._year
-            and match["annotations"] == self._annotations
-        )
-
-    def _classify_meta(self, data: Tuple[str, Any]) -> Optional[int]:
-        key, _ = data
-        if key == "images":
-            return 0
-        elif key == "annotations":
-            return 1
-        else:
-            return None
-
-    def _prepare_image(self, data: Tuple[str, BinaryIO]) -> Dict[str, Any]:
-        path, buffer = data
-        return dict(
-            path=path,
-            image=EncodedImage.from_file(buffer),
-        )
-
-    def _prepare_sample(
-        self,
-        data: Tuple[Tuple[List[Dict[str, Any]], Dict[str, Any]], Tuple[str, BinaryIO]],
-    ) -> Dict[str, Any]:
-        ann_data, image_data = data
-        anns, image_meta = ann_data
-
-        sample = self._prepare_image(image_data)
-        # this method is only called if we have annotations
-        annotations = cast(str, self._annotations)
-        sample.update(self._ANN_DECODERS[annotations](self, anns, image_meta))
-        return sample
-
-    def _datapipe(self, resource_dps: List[IterDataPipe]) -> IterDataPipe[Dict[str, Any]]:
-        images_dp, meta_dp = resource_dps
-
-        if self._annotations is None:
-            dp = hint_shuffling(images_dp)
-            dp = hint_sharding(dp)
-            dp = hint_shuffling(dp)
-            return Mapper(dp, self._prepare_image)
-
-        meta_dp = Filter(meta_dp, self._filter_meta_files)
-        meta_dp = JsonParser(meta_dp)
-        meta_dp = Mapper(meta_dp, getitem(1))
-        meta_dp: IterDataPipe[Dict[str, Dict[str, Any]]] = MappingIterator(meta_dp)
-        images_meta_dp, anns_meta_dp = Demultiplexer(
-            meta_dp,
-            2,
-            self._classify_meta,
-            drop_none=True,
-            buffer_size=INFINITE_BUFFER_SIZE,
-        )
-
-        images_meta_dp = Mapper(images_meta_dp, getitem(1))
-        images_meta_dp = UnBatcher(images_meta_dp)
-
-        anns_meta_dp = Mapper(anns_meta_dp, getitem(1))
-        anns_meta_dp = UnBatcher(anns_meta_dp)
-        anns_meta_dp = Grouper(anns_meta_dp, group_key_fn=getitem("image_id"), buffer_size=INFINITE_BUFFER_SIZE)
-        anns_meta_dp = hint_shuffling(anns_meta_dp)
-        anns_meta_dp = hint_sharding(anns_meta_dp)
-
-        anns_dp = IterKeyZipper(
-            anns_meta_dp,
-            images_meta_dp,
-            key_fn=getitem(0, "image_id"),
-            ref_key_fn=getitem("id"),
-            buffer_size=INFINITE_BUFFER_SIZE,
-        )
-        dp = IterKeyZipper(
-            anns_dp,
-            images_dp,
-            key_fn=getitem(1, "file_name"),
-            ref_key_fn=path_accessor("name"),
-            buffer_size=INFINITE_BUFFER_SIZE,
-        )
-        return Mapper(dp, self._prepare_sample)
-
-    def __len__(self) -> int:
-        return {
-            ("train", "2017"): defaultdict(lambda: 118_287, instances=117_266),
-            ("train", "2014"): defaultdict(lambda: 82_783, instances=82_081),
-            ("val", "2017"): defaultdict(lambda: 5_000, instances=4_952),
-            ("val", "2014"): defaultdict(lambda: 40_504, instances=40_137),
-        }[(self._split, self._year)][
-            self._annotations  # type: ignore[index]
-        ]
-
-    def _generate_categories(self) -> Tuple[Tuple[str, str]]:
-        self._annotations = "instances"
-        resources = self._resources()
-
-        dp = resources[1].load(self._root)
-        dp = Filter(dp, self._filter_meta_files)
-        dp = JsonParser(dp)
-
-        _, meta = next(iter(dp))
-        # List[Tuple[super_category, id, category]]
-        label_data = [cast(Tuple[str, int, str], tuple(info.values())) for info in meta["categories"]]
-
-        # COCO actually defines 91 categories, but only 80 of them have instances. Still, the category_id refers to the
-        # full set. To keep the labels dense, we fill the gaps with N/A. Note that there are only 10 gaps, so the total
-        # number of categories is 90 rather than 91.
-        _, ids, _ = zip(*label_data)
-        missing_ids = set(range(1, max(ids) + 1)) - set(ids)
-        label_data.extend([("N/A", id, "N/A") for id in missing_ids])
-
-        # We also add a background category to be used during segmentation.
-        label_data.append(("N/A", 0, "__background__"))
-
-        super_categories, _, categories = zip(*sorted(label_data, key=lambda info: info[1]))
-
-        return cast(Tuple[Tuple[str, str]], tuple(zip(categories, super_categories)))

torchvision/prototype/datasets/_builtin/country211.categories

-AD
-AE
-AF
-AG
-AI
-AL
-AM
-AO
-AQ
-AR
-AT
-AU
-AW
-AX
-AZ
-BA
-BB
-BD
-BE
-BF
-BG
-BH
-BJ
-BM
-BN
-BO
-BQ
-BR
-BS
-BT
-BW
-BY
-BZ
-CA
-CD
-CF
-CH
-CI
-CK
-CL
-CM
-CN
-CO
-CR
-CU
-CV
-CW
-CY
-CZ
-DE
-DK
-DM
-DO
-DZ
-EC
-EE
-EG
-ES
-ET
-FI
-FJ
-FK
-FO
-FR
-GA
-GB
-GD
-GE
-GF
-GG
-GH
-GI
-GL
-GM
-GP
-GR
-GS
-GT
-GU
-GY
-HK
-HN
-HR
-HT
-HU
-ID
-IE
-IL
-IM
-IN
-IQ
-IR
-IS
-IT
-JE
-JM
-JO
-JP
-KE
-KG
-KH
-KN
-KP
-KR
-KW
-KY
-KZ
-LA
-LB
-LC
-LI
-LK
-LR
-LT
-LU
-LV
-LY
-MA
-MC
-MD
-ME
-MF
-MG
-MK
-ML
-MM
-MN
-MO
-MQ
-MR
-MT
-MU
-MV
-MW
-MX
-MY
-MZ
-NA
-NC
-NG
-NI
-NL
-NO
-NP
-NZ
-OM
-PA
-PE
-PF
-PG
-PH
-PK
-PL
-PR
-PS
-PT
-PW
-PY
-QA
-RE
-RO
-RS
-RU
-RW
-SA
-SB
-SC
-SD
-SE
-SG
-SH
-SI
-SJ
-SK
-SL
-SM
-SN
-SO
-SS
-SV
-SX
-SY
-SZ
-TG
-TH
-TJ
-TL
-TM
-TN
-TO
-TR
-TT
-TW
-TZ
-UA
-UG
-US
-UY
-UZ
-VA
-VE
-VG
-VI
-VN
-VU
-WS
-XK
-YE
-ZA
-ZM
-ZW

torchvision/prototype/datasets/_builtin/country211.py

-import pathlib
-from typing import Any, Dict, List, Tuple, Union
-
-from torchdata.datapipes.iter import Filter, IterDataPipe, Mapper
-from torchvision.prototype.datasets.utils import Dataset, EncodedImage, HttpResource, OnlineResource
-from torchvision.prototype.datasets.utils._internal import (
-    hint_sharding,
-    hint_shuffling,
-    path_comparator,
-    read_categories_file,
-)
-from torchvision.prototype.tv_tensors import Label
-
-from .._api import register_dataset, register_info
-
-NAME = "country211"
-
-
-@register_info(NAME)
-def _info() -> Dict[str, Any]:
-    return dict(categories=read_categories_file(NAME))
-
-
-@register_dataset(NAME)
-class Country211(Dataset):
-    """
-    - **homepage**: https://github.com/openai/CLIP/blob/main/data/country211.md
-    """
-
-    def __init__(
-        self,
-        root: Union[str, pathlib.Path],
-        *,
-        split: str = "train",
-        skip_integrity_check: bool = False,
-    ) -> None:
-        self._split = self._verify_str_arg(split, "split", ("train", "val", "test"))
-        self._split_folder_name = "valid" if split == "val" else split
-
-        self._categories = _info()["categories"]
-
-        super().__init__(root, skip_integrity_check=skip_integrity_check)
-
-    def _resources(self) -> List[OnlineResource]:
-        return [
-            HttpResource(
-                "https://openaipublic.azureedge.net/clip/data/country211.tgz",
-                sha256="c011343cdc1296a8c31ff1d7129cf0b5e5b8605462cffd24f89266d6e6f4da3c",
-            )
-        ]
-
-    def _prepare_sample(self, data: Tuple[str, Any]) -> Dict[str, Any]:
-        path, buffer = data
-        category = pathlib.Path(path).parent.name
-        return dict(
-            label=Label.from_category(category, categories=self._categories),
-            path=path,
-            image=EncodedImage.from_file(buffer),
-        )
-
-    def _filter_split(self, data: Tuple[str, Any], *, split: str) -> bool:
-        return pathlib.Path(data[0]).parent.parent.name == split
-
-    def _datapipe(self, resource_dps: List[IterDataPipe]) -> IterDataPipe[Dict[str, Any]]:
-        dp = resource_dps[0]
-        dp = Filter(dp, path_comparator("parent.parent.name", self._split_folder_name))
-        dp = hint_shuffling(dp)
-        dp = hint_sharding(dp)
-        return Mapper(dp, self._prepare_sample)
-
-    def __len__(self) -> int:
-        return {
-            "train": 31_650,
-            "val": 10_550,
-            "test": 21_100,
-        }[self._split]
-
-    def _generate_categories(self) -> List[str]:
-        resources = self._resources()
-        dp = resources[0].load(self._root)
-        return sorted({pathlib.Path(path).parent.name for path, _ in dp})

torchvision/prototype/datasets/_builtin/cub200.categories

-Black_footed_Albatross
-Laysan_Albatross
-Sooty_Albatross
-Groove_billed_Ani
-Crested_Auklet
-Least_Auklet
-Parakeet_Auklet
-Rhinoceros_Auklet
-Brewer_Blackbird
-Red_winged_Blackbird
-Rusty_Blackbird
-Yellow_headed_Blackbird
-Bobolink
-Indigo_Bunting
-Lazuli_Bunting
-Painted_Bunting
-Cardinal
-Spotted_Catbird
-Gray_Catbird
-Yellow_breasted_Chat
-Eastern_Towhee
-Chuck_will_Widow
-Brandt_Cormorant
-Red_faced_Cormorant
-Pelagic_Cormorant
-Bronzed_Cowbird
-Shiny_Cowbird
-Brown_Creeper
-American_Crow
-Fish_Crow
-Black_billed_Cuckoo
-Mangrove_Cuckoo
-Yellow_billed_Cuckoo
-Gray_crowned_Rosy_Finch
-Purple_Finch
-Northern_Flicker
-Acadian_Flycatcher
-Great_Crested_Flycatcher
-Least_Flycatcher
-Olive_sided_Flycatcher
-Scissor_tailed_Flycatcher
-Vermilion_Flycatcher
-Yellow_bellied_Flycatcher
-Frigatebird
-Northern_Fulmar
-Gadwall
-American_Goldfinch
-European_Goldfinch
-Boat_tailed_Grackle
-Eared_Grebe
-Horned_Grebe
-Pied_billed_Grebe
-Western_Grebe
-Blue_Grosbeak
-Evening_Grosbeak
-Pine_Grosbeak
-Rose_breasted_Grosbeak
-Pigeon_Guillemot
-California_Gull
-Glaucous_winged_Gull
-Heermann_Gull
-Herring_Gull
-Ivory_Gull
-Ring_billed_Gull
-Slaty_backed_Gull
-Western_Gull
-Anna_Hummingbird
-Ruby_throated_Hummingbird
-Rufous_Hummingbird
-Green_Violetear
-Long_tailed_Jaeger
-Pomarine_Jaeger
-Blue_Jay
-Florida_Jay
-Green_Jay
-Dark_eyed_Junco
-Tropical_Kingbird
-Gray_Kingbird
-Belted_Kingfisher
-Green_Kingfisher
-Pied_Kingfisher
-Ringed_Kingfisher
-White_breasted_Kingfisher
-Red_legged_Kittiwake
-Horned_Lark
-Pacific_Loon
-Mallard
-Western_Meadowlark
-Hooded_Merganser
-Red_breasted_Merganser
-Mockingbird
-Nighthawk
-Clark_Nutcracker
-White_breasted_Nuthatch
-Baltimore_Oriole
-Hooded_Oriole
-Orchard_Oriole
-Scott_Oriole
-Ovenbird
-Brown_Pelican
-White_Pelican
-Western_Wood_Pewee
-Sayornis
-American_Pipit
-Whip_poor_Will
-Horned_Puffin
-Common_Raven
-White_necked_Raven
-American_Redstart
-Geococcyx
-Loggerhead_Shrike
-Great_Grey_Shrike
-Baird_Sparrow
-Black_throated_Sparrow
-Brewer_Sparrow
-Chipping_Sparrow
-Clay_colored_Sparrow
-House_Sparrow
-Field_Sparrow
-Fox_Sparrow
-Grasshopper_Sparrow
-Harris_Sparrow
-Henslow_Sparrow
-Le_Conte_Sparrow
-Lincoln_Sparrow
-Nelson_Sharp_tailed_Sparrow
-Savannah_Sparrow
-Seaside_Sparrow
-Song_Sparrow
-Tree_Sparrow
-Vesper_Sparrow
-White_crowned_Sparrow
-White_throated_Sparrow
-Cape_Glossy_Starling
-Bank_Swallow
-Barn_Swallow
-Cliff_Swallow
-Tree_Swallow
-Scarlet_Tanager
-Summer_Tanager
-Artic_Tern
-Black_Tern
-Caspian_Tern
-Common_Tern
-Elegant_Tern
-Forsters_Tern
-Least_Tern
-Green_tailed_Towhee
-Brown_Thrasher
-Sage_Thrasher
-Black_capped_Vireo
-Blue_headed_Vireo
-Philadelphia_Vireo
-Red_eyed_Vireo
-Warbling_Vireo
-White_eyed_Vireo
-Yellow_throated_Vireo
-Bay_breasted_Warbler
-Black_and_white_Warbler
-Black_throated_Blue_Warbler
-Blue_winged_Warbler
-Canada_Warbler
-Cape_May_Warbler
-Cerulean_Warbler
-Chestnut_sided_Warbler
-Golden_winged_Warbler
-Hooded_Warbler
-Kentucky_Warbler
-Magnolia_Warbler
-Mourning_Warbler
-Myrtle_Warbler
-Nashville_Warbler
-Orange_crowned_Warbler
-Palm_Warbler
-Pine_Warbler
-Prairie_Warbler
-Prothonotary_Warbler
-Swainson_Warbler
-Tennessee_Warbler
-Wilson_Warbler
-Worm_eating_Warbler
-Yellow_Warbler
-Northern_Waterthrush
-Louisiana_Waterthrush
-Bohemian_Waxwing
-Cedar_Waxwing
-American_Three_toed_Woodpecker
-Pileated_Woodpecker
-Red_bellied_Woodpecker
-Red_cockaded_Woodpecker
-Red_headed_Woodpecker
-Downy_Woodpecker
-Bewick_Wren
-Cactus_Wren
-Carolina_Wren
-House_Wren
-Marsh_Wren
-Rock_Wren
-Winter_Wren
-Common_Yellowthroat

torchvision/prototype/datasets/_builtin/cub200.py

-import csv
-import functools
-import pathlib
-from typing import Any, BinaryIO, Callable, Dict, List, Optional, Tuple, Union
-
-import torch
-from torchdata.datapipes.iter import (
-    CSVDictParser,
-    CSVParser,
-    Demultiplexer,
-    Filter,
-    IterDataPipe,
-    IterKeyZipper,
-    LineReader,
-    Mapper,
-)
-from torchdata.datapipes.map import IterToMapConverter
-from torchvision.prototype.datasets.utils import Dataset, EncodedImage, GDriveResource, OnlineResource
-from torchvision.prototype.datasets.utils._internal import (
-    getitem,
-    hint_sharding,
-    hint_shuffling,
-    INFINITE_BUFFER_SIZE,
-    path_accessor,
-    path_comparator,
-    read_categories_file,
-    read_mat,
-)
-from torchvision.prototype.tv_tensors import Label
-from torchvision.tv_tensors import BoundingBoxes
-
-from .._api import register_dataset, register_info
-
-csv.register_dialect("cub200", delimiter=" ")
-
-
-NAME = "cub200"
-
-
-@register_info(NAME)
-def _info() -> Dict[str, Any]:
-    return dict(categories=read_categories_file(NAME))
-
-
-@register_dataset(NAME)
-class CUB200(Dataset):
-    """
-    - **homepage**: http://www.vision.caltech.edu/visipedia/CUB-200.html
-    """
-
-    def __init__(
-        self,
-        root: Union[str, pathlib.Path],
-        *,
-        split: str = "train",
-        year: str = "2011",
-        skip_integrity_check: bool = False,
-    ) -> None:
-        self._split = self._verify_str_arg(split, "split", ("train", "test"))
-        self._year = self._verify_str_arg(year, "year", ("2010", "2011"))
-
-        self._categories = _info()["categories"]
-
-        super().__init__(
-            root,
-            # TODO: this will only be available after https://github.com/pytorch/vision/pull/5473
-            # dependencies=("scipy",),
-            skip_integrity_check=skip_integrity_check,
-        )
-
-    def _resources(self) -> List[OnlineResource]:
-        if self._year == "2011":
-            archive = GDriveResource(
-                "1hbzc_P1FuxMkcabkgn9ZKinBwW683j45",
-                file_name="CUB_200_2011.tgz",
-                sha256="0c685df5597a8b24909f6a7c9db6d11e008733779a671760afef78feb49bf081",
-                preprocess="decompress",
-            )
-            segmentations = GDriveResource(
-                "1EamOKGLoTuZdtcVYbHMWNpkn3iAVj8TP",
-                file_name="segmentations.tgz",
-                sha256="dc77f6cffea0cbe2e41d4201115c8f29a6320ecb04fffd2444f51b8066e4b84f",
-                preprocess="decompress",
-            )
-            return [archive, segmentations]
-        else:  # self._year == "2010"
-            split = GDriveResource(
-                "1vZuZPqha0JjmwkdaS_XtYryE3Jf5Q1AC",
-                file_name="lists.tgz",
-                sha256="aeacbd5e3539ae84ea726e8a266a9a119c18f055cd80f3836d5eb4500b005428",
-                preprocess="decompress",
-            )
-            images = GDriveResource(
-                "1GDr1OkoXdhaXWGA8S3MAq3a522Tak-nx",
-                file_name="images.tgz",
-                sha256="2a6d2246bbb9778ca03aa94e2e683ccb4f8821a36b7f235c0822e659d60a803e",
-                preprocess="decompress",
-            )
-            anns = GDriveResource(
-                "16NsbTpMs5L6hT4hUJAmpW2u7wH326WTR",
-                file_name="annotations.tgz",
-                sha256="c17b7841c21a66aa44ba8fe92369cc95dfc998946081828b1d7b8a4b716805c1",
-                preprocess="decompress",
-            )
-            return [split, images, anns]
-
-    def _2011_classify_archive(self, data: Tuple[str, Any]) -> Optional[int]:
-        path = pathlib.Path(data[0])
-        if path.parents[1].name == "images":
-            return 0
-        elif path.name == "train_test_split.txt":
-            return 1
-        elif path.name == "images.txt":
-            return 2
-        elif path.name == "bounding_boxes.txt":
-            return 3
-        else:
-            return None
-
-    def _2011_extract_file_name(self, rel_posix_path: str) -> str:
-        return rel_posix_path.rsplit("/", maxsplit=1)[1]
-
-    def _2011_filter_split(self, row: List[str]) -> bool:
-        _, split_id = row
-        return {
-            "0": "test",
-            "1": "train",
-        }[split_id] == self._split
-
-    def _2011_segmentation_key(self, data: Tuple[str, Any]) -> str:
-        path = pathlib.Path(data[0])
-        return path.with_suffix(".jpg").name
-
-    def _2011_prepare_ann(
-        self, data: Tuple[str, Tuple[List[str], Tuple[str, BinaryIO]]], spatial_size: Tuple[int, int]
-    ) -> Dict[str, Any]:
-        _, (bounding_boxes_data, segmentation_data) = data
-        segmentation_path, segmentation_buffer = segmentation_data
-        return dict(
-            bounding_boxes=BoundingBoxes(
-                [float(part) for part in bounding_boxes_data[1:]], format="xywh", spatial_size=spatial_size
-            ),
-            segmentation_path=segmentation_path,
-            segmentation=EncodedImage.from_file(segmentation_buffer),
-        )
-
-    def _2010_split_key(self, data: str) -> str:
-        return data.rsplit("/", maxsplit=1)[1]
-
-    def _2010_anns_key(self, data: Tuple[str, BinaryIO]) -> Tuple[str, Tuple[str, BinaryIO]]:
-        path = pathlib.Path(data[0])
-        return path.with_suffix(".jpg").name, data
-
-    def _2010_prepare_ann(
-        self, data: Tuple[str, Tuple[str, BinaryIO]], spatial_size: Tuple[int, int]
-    ) -> Dict[str, Any]:
-        _, (path, buffer) = data
-        content = read_mat(buffer)
-        return dict(
-            ann_path=path,
-            bounding_boxes=BoundingBoxes(
-                [int(content["bbox"][coord]) for coord in ("left", "bottom", "right", "top")],
-                format="xyxy",
-                spatial_size=spatial_size,
-            ),
-            segmentation=torch.as_tensor(content["seg"]),
-        )
-
-    def _prepare_sample(
-        self,
-        data: Tuple[Tuple[str, Tuple[str, BinaryIO]], Any],
-        *,
-        prepare_ann_fn: Callable[[Any, Tuple[int, int]], Dict[str, Any]],
-    ) -> Dict[str, Any]:
-        data, anns_data = data
-        _, image_data = data
-        path, buffer = image_data
-
-        image = EncodedImage.from_file(buffer)
-
-        return dict(
-            prepare_ann_fn(anns_data, image.spatial_size),
-            image=image,
-            label=Label(
-                int(pathlib.Path(path).parent.name.rsplit(".", 1)[0]) - 1,
-                categories=self._categories,
-            ),
-        )
-
-    def _datapipe(self, resource_dps: List[IterDataPipe]) -> IterDataPipe[Dict[str, Any]]:
-        prepare_ann_fn: Callable
-        if self._year == "2011":
-            archive_dp, segmentations_dp = resource_dps
-            images_dp, split_dp, image_files_dp, bounding_boxes_dp = Demultiplexer(
-                archive_dp, 4, self._2011_classify_archive, drop_none=True, buffer_size=INFINITE_BUFFER_SIZE
-            )
-
-            image_files_dp = CSVParser(image_files_dp, dialect="cub200")
-            image_files_dp = Mapper(image_files_dp, self._2011_extract_file_name, input_col=1)
-            image_files_map = IterToMapConverter(image_files_dp)
-
-            split_dp = CSVParser(split_dp, dialect="cub200")
-            split_dp = Filter(split_dp, self._2011_filter_split)
-            split_dp = Mapper(split_dp, getitem(0))
-            split_dp = Mapper(split_dp, image_files_map.__getitem__)
-
-            bounding_boxes_dp = CSVParser(bounding_boxes_dp, dialect="cub200")
-            bounding_boxes_dp = Mapper(bounding_boxes_dp, image_files_map.__getitem__, input_col=0)
-
-            anns_dp = IterKeyZipper(
-                bounding_boxes_dp,
-                segmentations_dp,
-                key_fn=getitem(0),
-                ref_key_fn=self._2011_segmentation_key,
-                keep_key=True,
-                buffer_size=INFINITE_BUFFER_SIZE,
-            )
-
-            prepare_ann_fn = self._2011_prepare_ann
-        else:  # self._year == "2010"
-            split_dp, images_dp, anns_dp = resource_dps
-
-            split_dp = Filter(split_dp, path_comparator("name", f"{self._split}.txt"))
-            split_dp = LineReader(split_dp, decode=True, return_path=False)
-            split_dp = Mapper(split_dp, self._2010_split_key)
-
-            anns_dp = Mapper(anns_dp, self._2010_anns_key)
-
-            prepare_ann_fn = self._2010_prepare_ann
-
-        split_dp = hint_shuffling(split_dp)
-        split_dp = hint_sharding(split_dp)
-
-        dp = IterKeyZipper(
-            split_dp,
-            images_dp,
-            getitem(),
-            path_accessor("name"),
-            buffer_size=INFINITE_BUFFER_SIZE,
-        )
-        dp = IterKeyZipper(
-            dp,
-            anns_dp,
-            getitem(0),
-            buffer_size=INFINITE_BUFFER_SIZE,
-        )
-        return Mapper(dp, functools.partial(self._prepare_sample, prepare_ann_fn=prepare_ann_fn))
-
-    def __len__(self) -> int:
-        return {
-            ("train", "2010"): 3_000,
-            ("test", "2010"): 3_033,
-            ("train", "2011"): 5_994,
-            ("test", "2011"): 5_794,
-        }[(self._split, self._year)]
-
-    def _generate_categories(self) -> List[str]:
-        self._year = "2011"
-        resources = self._resources()
-
-        dp = resources[0].load(self._root)
-        dp = Filter(dp, path_comparator("name", "classes.txt"))
-        dp = CSVDictParser(dp, fieldnames=("label", "category"), dialect="cub200")
-
-        return [row["category"].split(".")[1] for row in dp]

torchvision/prototype/datasets/_builtin/dtd.categories

-banded
-blotchy
-braided
-bubbly
-bumpy
-chequered
-cobwebbed
-cracked
-crosshatched
-crystalline
-dotted
-fibrous
-flecked
-freckled
-frilly
-gauzy
-grid
-grooved
-honeycombed
-interlaced
-knitted
-lacelike
-lined
-marbled
-matted
-meshed
-paisley
-perforated
-pitted
-pleated
-polka-dotted
-porous
-potholed
-scaly
-smeared
-spiralled
-sprinkled
-stained
-stratified
-striped
-studded
-swirly
-veined
-waffled
-woven
-wrinkled
-zigzagged

torchvision/prototype/datasets/_builtin/dtd.py

-import enum
-import pathlib
-from typing import Any, BinaryIO, Dict, List, Optional, Tuple, Union
-
-from torchdata.datapipes.iter import CSVParser, Demultiplexer, Filter, IterDataPipe, IterKeyZipper, LineReader, Mapper
-from torchvision.prototype.datasets.utils import Dataset, EncodedImage, HttpResource, OnlineResource
-from torchvision.prototype.datasets.utils._internal import (
-    getitem,
-    hint_sharding,
-    hint_shuffling,
-    INFINITE_BUFFER_SIZE,
-    path_comparator,
-    read_categories_file,
-)
-from torchvision.prototype.tv_tensors import Label
-
-from .._api import register_dataset, register_info
-
-
-NAME = "dtd"
-
-
-class DTDDemux(enum.IntEnum):
-    SPLIT = 0
-    JOINT_CATEGORIES = 1
-    IMAGES = 2
-
-
-@register_info(NAME)
-def _info() -> Dict[str, Any]:
-    return dict(categories=read_categories_file(NAME))
-
-
-@register_dataset(NAME)
-class DTD(Dataset):
-    """DTD Dataset.
-    homepage="https://www.robots.ox.ac.uk/~vgg/data/dtd/",
-    """
-
-    def __init__(
-        self,
-        root: Union[str, pathlib.Path],
-        *,
-        split: str = "train",
-        fold: int = 1,
-        skip_validation_check: bool = False,
-    ) -> None:
-        self._split = self._verify_str_arg(split, "split", {"train", "val", "test"})
-
-        if not (1 <= fold <= 10):
-            raise ValueError(f"The fold parameter should be an integer in [1, 10]. Got {fold}")
-        self._fold = fold
-
-        self._categories = _info()["categories"]
-
-        super().__init__(root, skip_integrity_check=skip_validation_check)
-
-    def _resources(self) -> List[OnlineResource]:
-        archive = HttpResource(
-            "https://www.robots.ox.ac.uk/~vgg/data/dtd/download/dtd-r1.0.1.tar.gz",
-            sha256="e42855a52a4950a3b59612834602aa253914755c95b0cff9ead6d07395f8e205",
-            preprocess="decompress",
-        )
-        return [archive]
-
-    def _classify_archive(self, data: Tuple[str, Any]) -> Optional[int]:
-        path = pathlib.Path(data[0])
-        if path.parent.name == "labels":
-            if path.name == "labels_joint_anno.txt":
-                return DTDDemux.JOINT_CATEGORIES
-
-            return DTDDemux.SPLIT
-        elif path.parents[1].name == "images":
-            return DTDDemux.IMAGES
-        else:
-            return None
-
-    def _image_key_fn(self, data: Tuple[str, Any]) -> str:
-        path = pathlib.Path(data[0])
-        # The split files contain hardcoded posix paths for the images, e.g. banded/banded_0001.jpg
-        return str(path.relative_to(path.parents[1]).as_posix())
-
-    def _prepare_sample(self, data: Tuple[Tuple[str, List[str]], Tuple[str, BinaryIO]]) -> Dict[str, Any]:
-        (_, joint_categories_data), image_data = data
-        _, *joint_categories = joint_categories_data
-        path, buffer = image_data
-
-        category = pathlib.Path(path).parent.name
-
-        return dict(
-            joint_categories={category for category in joint_categories if category},
-            label=Label.from_category(category, categories=self._categories),
-            path=path,
-            image=EncodedImage.from_file(buffer),
-        )
-
-    def _datapipe(self, resource_dps: List[IterDataPipe]) -> IterDataPipe[Dict[str, Any]]:
-        archive_dp = resource_dps[0]
-
-        splits_dp, joint_categories_dp, images_dp = Demultiplexer(
-            archive_dp, 3, self._classify_archive, drop_none=True, buffer_size=INFINITE_BUFFER_SIZE
-        )
-
-        splits_dp = Filter(splits_dp, path_comparator("name", f"{self._split}{self._fold}.txt"))
-        splits_dp = LineReader(splits_dp, decode=True, return_path=False)
-        splits_dp = hint_shuffling(splits_dp)
-        splits_dp = hint_sharding(splits_dp)
-
-        joint_categories_dp = CSVParser(joint_categories_dp, delimiter=" ")
-
-        dp = IterKeyZipper(
-            splits_dp,
-            joint_categories_dp,
-            key_fn=getitem(),
-            ref_key_fn=getitem(0),
-            buffer_size=INFINITE_BUFFER_SIZE,
-        )
-        dp = IterKeyZipper(
-            dp,
-            images_dp,
-            key_fn=getitem(0),
-            ref_key_fn=self._image_key_fn,
-            buffer_size=INFINITE_BUFFER_SIZE,
-        )
-        return Mapper(dp, self._prepare_sample)
-
-    def _filter_images(self, data: Tuple[str, Any]) -> bool:
-        return self._classify_archive(data) == DTDDemux.IMAGES
-
-    def _generate_categories(self) -> List[str]:
-        resources = self._resources()
-
-        dp = resources[0].load(self._root)
-        dp = Filter(dp, self._filter_images)
-
-        return sorted({pathlib.Path(path).parent.name for path, _ in dp})
-
-    def __len__(self) -> int:
-        return 1_880  # All splits have the same length

torchvision/prototype/datasets/_builtin/eurosat.py

-import pathlib
-from typing import Any, Dict, List, Tuple, Union
-
-from torchdata.datapipes.iter import IterDataPipe, Mapper
-from torchvision.prototype.datasets.utils import Dataset, EncodedImage, HttpResource, OnlineResource
-from torchvision.prototype.datasets.utils._internal import hint_sharding, hint_shuffling
-from torchvision.prototype.tv_tensors import Label
-
-from .._api import register_dataset, register_info
-
-NAME = "eurosat"
-
-
-@register_info(NAME)
-def _info() -> Dict[str, Any]:
-    return dict(
-        categories=(
-            "AnnualCrop",
-            "Forest",
-            "HerbaceousVegetation",
-            "Highway",
-            "Industrial",
-            "Pasture",
-            "PermanentCrop",
-            "Residential",
-            "River",
-            "SeaLake",
-        )
-    )
-
-
-@register_dataset(NAME)
-class EuroSAT(Dataset):
-    """EuroSAT Dataset.
-    homepage="https://github.com/phelber/eurosat",
-    """
-
-    def __init__(self, root: Union[str, pathlib.Path], *, skip_integrity_check: bool = False) -> None:
-        self._categories = _info()["categories"]
-        super().__init__(root, skip_integrity_check=skip_integrity_check)
-
-    def _resources(self) -> List[OnlineResource]:
-        return [
-            HttpResource(
-                "https://madm.dfki.de/files/sentinel/EuroSAT.zip",
-                sha256="8ebea626349354c5328b142b96d0430e647051f26efc2dc974c843f25ecf70bd",
-            )
-        ]
-
-    def _prepare_sample(self, data: Tuple[str, Any]) -> Dict[str, Any]:
-        path, buffer = data
-        category = pathlib.Path(path).parent.name
-        return dict(
-            label=Label.from_category(category, categories=self._categories),
-            path=path,
-            image=EncodedImage.from_file(buffer),
-        )
-
-    def _datapipe(self, resource_dps: List[IterDataPipe]) -> IterDataPipe[Dict[str, Any]]:
-        dp = resource_dps[0]
-        dp = hint_shuffling(dp)
-        dp = hint_sharding(dp)
-        return Mapper(dp, self._prepare_sample)
-
-    def __len__(self) -> int:
-        return 27_000

torchvision/prototype/datasets/_builtin/fer2013.py

-import pathlib
-from typing import Any, Dict, List, Union
-
-import torch
-from torchdata.datapipes.iter import CSVDictParser, IterDataPipe, Mapper
-from torchvision.prototype.datasets.utils import Dataset, KaggleDownloadResource, OnlineResource
-from torchvision.prototype.datasets.utils._internal import hint_sharding, hint_shuffling
-from torchvision.prototype.tv_tensors import Label
-from torchvision.tv_tensors import Image
-
-from .._api import register_dataset, register_info
-
-NAME = "fer2013"
-
-
-@register_info(NAME)
-def _info() -> Dict[str, Any]:
-    return dict(categories=("angry", "disgust", "fear", "happy", "sad", "surprise", "neutral"))
-
-
-@register_dataset(NAME)
-class FER2013(Dataset):
-    """FER 2013 Dataset
-    homepage="https://www.kaggle.com/c/challenges-in-representation-learning-facial-expression-recognition-challenge"
-    """
-
-    def __init__(
-        self, root: Union[str, pathlib.Path], *, split: str = "train", skip_integrity_check: bool = False
-    ) -> None:
-        self._split = self._verify_str_arg(split, "split", {"train", "test"})
-        self._categories = _info()["categories"]
-
-        super().__init__(root, skip_integrity_check=skip_integrity_check)
-
-    _CHECKSUMS = {
-        "train": "a2b7c9360cc0b38d21187e5eece01c2799fce5426cdeecf746889cc96cda2d10",
-        "test": "dec8dfe8021e30cd6704b85ec813042b4a5d99d81cb55e023291a94104f575c3",
-    }
-
-    def _resources(self) -> List[OnlineResource]:
-        archive = KaggleDownloadResource(
-            "https://www.kaggle.com/c/challenges-in-representation-learning-facial-expression-recognition-challenge",
-            file_name=f"{self._split}.csv.zip",
-            sha256=self._CHECKSUMS[self._split],
-        )
-        return [archive]
-
-    def _prepare_sample(self, data: Dict[str, Any]) -> Dict[str, Any]:
-        label_id = data.get("emotion")
-
-        return dict(
-            image=Image(torch.tensor([int(idx) for idx in data["pixels"].split()], dtype=torch.uint8).reshape(48, 48)),
-            label=Label(int(label_id), categories=self._categories) if label_id is not None else None,
-        )
-
-    def _datapipe(self, resource_dps: List[IterDataPipe]) -> IterDataPipe[Dict[str, Any]]:
-        dp = resource_dps[0]
-        dp = CSVDictParser(dp)
-        dp = hint_shuffling(dp)
-        dp = hint_sharding(dp)
-        return Mapper(dp, self._prepare_sample)
-
-    def __len__(self) -> int:
-        return 28_709 if self._split == "train" else 3_589

torchvision/prototype/datasets/_builtin/food101.categories

-apple_pie
-baby_back_ribs
-baklava
-beef_carpaccio
-beef_tartare
-beet_salad
-beignets
-bibimbap
-bread_pudding
-breakfast_burrito
-bruschetta
-caesar_salad
-cannoli
-caprese_salad
-carrot_cake
-ceviche
-cheesecake
-cheese_plate
-chicken_curry
-chicken_quesadilla
-chicken_wings
-chocolate_cake
-chocolate_mousse
-churros
-clam_chowder
-club_sandwich
-crab_cakes
-creme_brulee
-croque_madame
-cup_cakes
-deviled_eggs
-donuts
-dumplings
-edamame
-eggs_benedict
-escargots
-falafel
-filet_mignon
-fish_and_chips
-foie_gras
-french_fries
-french_onion_soup
-french_toast
-fried_calamari
-fried_rice
-frozen_yogurt
-garlic_bread
-gnocchi
-greek_salad
-grilled_cheese_sandwich
-grilled_salmon
-guacamole
-gyoza
-hamburger
-hot_and_sour_soup
-hot_dog
-huevos_rancheros
-hummus
-ice_cream
-lasagna
-lobster_bisque
-lobster_roll_sandwich
-macaroni_and_cheese
-macarons
-miso_soup
-mussels
-nachos
-omelette
-onion_rings
-oysters
-pad_thai
-paella
-pancakes
-panna_cotta
-peking_duck
-pho
-pizza
-pork_chop
-poutine
-prime_rib
-pulled_pork_sandwich
-ramen
-ravioli
-red_velvet_cake
-risotto
-samosa
-sashimi
-scallops
-seaweed_salad
-shrimp_and_grits
-spaghetti_bolognese
-spaghetti_carbonara
-spring_rolls
-steak
-strawberry_shortcake
-sushi
-tacos
-takoyaki
-tiramisu
-tuna_tartare
-waffles

torchvision/prototype/datasets/_builtin/food101.py

-from pathlib import Path
-from typing import Any, BinaryIO, Dict, List, Optional, Tuple, Union
-
-from torchdata.datapipes.iter import Demultiplexer, Filter, IterDataPipe, IterKeyZipper, LineReader, Mapper
-from torchvision.prototype.datasets.utils import Dataset, EncodedImage, HttpResource, OnlineResource
-from torchvision.prototype.datasets.utils._internal import (
-    getitem,
-    hint_sharding,
-    hint_shuffling,
-    INFINITE_BUFFER_SIZE,
-    path_comparator,
-    read_categories_file,
-)
-from torchvision.prototype.tv_tensors import Label
-
-from .._api import register_dataset, register_info
-
-
-NAME = "food101"
-
-
-@register_info(NAME)
-def _info() -> Dict[str, Any]:
-    return dict(categories=read_categories_file(NAME))
-
-
-@register_dataset(NAME)
-class Food101(Dataset):
-    """Food 101 dataset
-    homepage="https://data.vision.ee.ethz.ch/cvl/datasets_extra/food-101",
-    """
-
-    def __init__(self, root: Union[str, Path], *, split: str = "train", skip_integrity_check: bool = False) -> None:
-        self._split = self._verify_str_arg(split, "split", {"train", "test"})
-        self._categories = _info()["categories"]
-
-        super().__init__(root, skip_integrity_check=skip_integrity_check)
-
-    def _resources(self) -> List[OnlineResource]:
-        return [
-            HttpResource(
-                url="http://data.vision.ee.ethz.ch/cvl/food-101.tar.gz",
-                sha256="d97d15e438b7f4498f96086a4f7e2fa42a32f2712e87d3295441b2b6314053a4",
-                preprocess="decompress",
-            )
-        ]
-
-    def _classify_archive(self, data: Tuple[str, Any]) -> Optional[int]:
-        path = Path(data[0])
-        if path.parents[1].name == "images":
-            return 0
-        elif path.parents[0].name == "meta":
-            return 1
-        else:
-            return None
-
-    def _prepare_sample(self, data: Tuple[str, Tuple[str, BinaryIO]]) -> Dict[str, Any]:
-        id, (path, buffer) = data
-        return dict(
-            label=Label.from_category(id.split("/", 1)[0], categories=self._categories),
-            path=path,
-            image=EncodedImage.from_file(buffer),
-        )
-
-    def _image_key(self, data: Tuple[str, Any]) -> str:
-        path = Path(data[0])
-        return path.relative_to(path.parents[1]).with_suffix("").as_posix()
-
-    def _datapipe(self, resource_dps: List[IterDataPipe]) -> IterDataPipe[Dict[str, Any]]:
-        archive_dp = resource_dps[0]
-        images_dp, split_dp = Demultiplexer(
-            archive_dp, 2, self._classify_archive, drop_none=True, buffer_size=INFINITE_BUFFER_SIZE
-        )
-        split_dp = Filter(split_dp, path_comparator("name", f"{self._split}.txt"))
-        split_dp = LineReader(split_dp, decode=True, return_path=False)
-        split_dp = hint_sharding(split_dp)
-        split_dp = hint_shuffling(split_dp)
-
-        dp = IterKeyZipper(
-            split_dp,
-            images_dp,
-            key_fn=getitem(),
-            ref_key_fn=self._image_key,
-            buffer_size=INFINITE_BUFFER_SIZE,
-        )
-
-        return Mapper(dp, self._prepare_sample)
-
-    def _generate_categories(self) -> List[str]:
-        resources = self._resources()
-        dp = resources[0].load(self._root)
-        dp = Filter(dp, path_comparator("name", "classes.txt"))
-        dp = LineReader(dp, decode=True, return_path=False)
-        return list(dp)
-
-    def __len__(self) -> int:
-        return 75_750 if self._split == "train" else 25_250

torchvision/prototype/datasets/_builtin/gtsrb.py

-import pathlib
-from typing import Any, Dict, List, Optional, Tuple, Union
-
-from torchdata.datapipes.iter import CSVDictParser, Demultiplexer, Filter, IterDataPipe, Mapper, Zipper
-from torchvision.prototype.datasets.utils import Dataset, EncodedImage, HttpResource, OnlineResource
-from torchvision.prototype.datasets.utils._internal import (
-    hint_sharding,
-    hint_shuffling,
-    INFINITE_BUFFER_SIZE,
-    path_comparator,
-)
-from torchvision.prototype.tv_tensors import Label
-from torchvision.tv_tensors import BoundingBoxes
-
-from .._api import register_dataset, register_info
-
-NAME = "gtsrb"
-
-
-@register_info(NAME)
-def _info() -> Dict[str, Any]:
-    return dict(
-        categories=[f"{label:05d}" for label in range(43)],
-    )
-
-
-@register_dataset(NAME)
-class GTSRB(Dataset):
-    """GTSRB Dataset
-
-    homepage="https://benchmark.ini.rub.de"
-    """
-
-    def __init__(
-        self, root: Union[str, pathlib.Path], *, split: str = "train", skip_integrity_check: bool = False
-    ) -> None:
-        self._split = self._verify_str_arg(split, "split", {"train", "test"})
-        self._categories = _info()["categories"]
-        super().__init__(root, skip_integrity_check=skip_integrity_check)
-
-    _URL_ROOT = "https://sid.erda.dk/public/archives/daaeac0d7ce1152aea9b61d9f1e19370/"
-    _URLS = {
-        "train": f"{_URL_ROOT}GTSRB-Training_fixed.zip",
-        "test": f"{_URL_ROOT}GTSRB_Final_Test_Images.zip",
-        "test_ground_truth": f"{_URL_ROOT}GTSRB_Final_Test_GT.zip",
-    }
-    _CHECKSUMS = {
-        "train": "df4144942083645bd60b594de348aa6930126c3e0e5de09e39611630abf8455a",
-        "test": "48ba6fab7e877eb64eaf8de99035b0aaecfbc279bee23e35deca4ac1d0a837fa",
-        "test_ground_truth": "f94e5a7614d75845c74c04ddb26b8796b9e483f43541dd95dd5b726504e16d6d",
-    }
-
-    def _resources(self) -> List[OnlineResource]:
-        rsrcs: List[OnlineResource] = [HttpResource(self._URLS[self._split], sha256=self._CHECKSUMS[self._split])]
-
-        if self._split == "test":
-            rsrcs.append(
-                HttpResource(
-                    self._URLS["test_ground_truth"],
-                    sha256=self._CHECKSUMS["test_ground_truth"],
-                )
-            )
-
-        return rsrcs
-
-    def _classify_train_archive(self, data: Tuple[str, Any]) -> Optional[int]:
-        path = pathlib.Path(data[0])
-        if path.suffix == ".ppm":
-            return 0
-        elif path.suffix == ".csv":
-            return 1
-        else:
-            return None
-
-    def _prepare_sample(self, data: Tuple[Tuple[str, Any], Dict[str, Any]]) -> Dict[str, Any]:
-        (path, buffer), csv_info = data
-        label = int(csv_info["ClassId"])
-
-        bounding_boxes = BoundingBoxes(
-            [int(csv_info[k]) for k in ("Roi.X1", "Roi.Y1", "Roi.X2", "Roi.Y2")],
-            format="xyxy",
-            spatial_size=(int(csv_info["Height"]), int(csv_info["Width"])),
-        )
-
-        return {
-            "path": path,
-            "image": EncodedImage.from_file(buffer),
-            "label": Label(label, categories=self._categories),
-            "bounding_boxes": bounding_boxes,
-        }
-
-    def _datapipe(self, resource_dps: List[IterDataPipe]) -> IterDataPipe[Dict[str, Any]]:
-        if self._split == "train":
-            images_dp, ann_dp = Demultiplexer(
-                resource_dps[0], 2, self._classify_train_archive, drop_none=True, buffer_size=INFINITE_BUFFER_SIZE
-            )
-        else:
-            images_dp, ann_dp = resource_dps
-            images_dp = Filter(images_dp, path_comparator("suffix", ".ppm"))
-
-        # The order of the image files in the .zip archives perfectly match the order of the entries in the
-        # (possibly concatenated) .csv files. So we're able to use Zipper here instead of a IterKeyZipper.
-        ann_dp = CSVDictParser(ann_dp, delimiter=";")
-        dp = Zipper(images_dp, ann_dp)
-
-        dp = hint_shuffling(dp)
-        dp = hint_sharding(dp)
-
-        return Mapper(dp, self._prepare_sample)
-
-    def __len__(self) -> int:
-        return 26_640 if self._split == "train" else 12_630

torchvision/prototype/datasets/_builtin/imagenet.py

-import enum
-import pathlib
-import re
-
-from typing import Any, BinaryIO, cast, Dict, Iterator, List, Match, Optional, Tuple, Union
-
-from torchdata.datapipes.iter import (
-    Demultiplexer,
-    Enumerator,
-    Filter,
-    IterDataPipe,
-    IterKeyZipper,
-    LineReader,
-    Mapper,
-    TarArchiveLoader,
-)
-from torchdata.datapipes.map import IterToMapConverter
-from torchvision.prototype.datasets.utils import Dataset, EncodedImage, ManualDownloadResource, OnlineResource
-from torchvision.prototype.datasets.utils._internal import (
-    getitem,
-    hint_sharding,
-    hint_shuffling,
-    INFINITE_BUFFER_SIZE,
-    path_accessor,
-    read_categories_file,
-    read_mat,
-)
-from torchvision.prototype.tv_tensors import Label
-
-from .._api import register_dataset, register_info
-
-NAME = "imagenet"
-
-
-@register_info(NAME)
-def _info() -> Dict[str, Any]:
-    categories, wnids = zip(*read_categories_file(NAME))
-    return dict(categories=categories, wnids=wnids)
-
-
-class ImageNetResource(ManualDownloadResource):
-    def __init__(self, **kwargs: Any) -> None:
-        super().__init__("Register on https://image-net.org/ and follow the instructions there.", **kwargs)
-
-
-class ImageNetDemux(enum.IntEnum):
-    META = 0
-    LABEL = 1
-
-
-class CategoryAndWordNetIDExtractor(IterDataPipe):
-    # Although the WordNet IDs (wnids) are unique, the corresponding categories are not. For example, both n02012849
-    # and n03126707 are labeled 'crane' while the first means the bird and the latter means the construction equipment
-    _WNID_MAP = {
-        "n03126707": "construction crane",
-        "n03710721": "tank suit",
-    }
-
-    def __init__(self, datapipe: IterDataPipe[Tuple[str, BinaryIO]]) -> None:
-        self.datapipe = datapipe
-
-    def __iter__(self) -> Iterator[Tuple[str, str]]:
-        for _, stream in self.datapipe:
-            synsets = read_mat(stream, squeeze_me=True)["synsets"]
-            for _, wnid, category, _, num_children, *_ in synsets:
-                if num_children > 0:
-                    # we are looking at a superclass that has no direct instance
-                    continue
-
-                yield self._WNID_MAP.get(wnid, category.split(",", 1)[0]), wnid
-
-
-@register_dataset(NAME)
-class ImageNet(Dataset):
-    """
-    - **homepage**: https://www.image-net.org/
-    """
-
-    def __init__(
-        self,
-        root: Union[str, pathlib.Path],
-        *,
-        split: str = "train",
-        skip_integrity_check: bool = False,
-    ) -> None:
-        self._split = self._verify_str_arg(split, "split", {"train", "val", "test"})
-
-        info = _info()
-        categories, wnids = info["categories"], info["wnids"]
-        self._categories = categories
-        self._wnids = wnids
-        self._wnid_to_category = dict(zip(wnids, categories))
-
-        super().__init__(root, skip_integrity_check=skip_integrity_check)
-
-    _IMAGES_CHECKSUMS = {
-        "train": "b08200a27a8e34218a0e58fde36b0fe8f73bc377f4acea2d91602057c3ca45bb",
-        "val": "c7e06a6c0baccf06d8dbeb6577d71efff84673a5dbdd50633ab44f8ea0456ae0",
-        "test_v10102019": "9cf7f8249639510f17d3d8a0deb47cd22a435886ba8e29e2b3223e65a4079eb4",
-    }
-
-    def _resources(self) -> List[OnlineResource]:
-        name = "test_v10102019" if self._split == "test" else self._split
-        images = ImageNetResource(
-            file_name=f"ILSVRC2012_img_{name}.tar",
-            sha256=self._IMAGES_CHECKSUMS[name],
-        )
-        resources: List[OnlineResource] = [images]
-
-        if self._split == "val":
-            devkit = ImageNetResource(
-                file_name="ILSVRC2012_devkit_t12.tar.gz",
-                sha256="b59243268c0d266621fd587d2018f69e906fb22875aca0e295b48cafaa927953",
-            )
-            resources.append(devkit)
-
-        return resources
-
-    _TRAIN_IMAGE_NAME_PATTERN = re.compile(r"(?P<wnid>n\d{8})_\d+[.]JPEG")
-
-    def _prepare_train_data(self, data: Tuple[str, BinaryIO]) -> Tuple[Tuple[Label, str], Tuple[str, BinaryIO]]:
-        path = pathlib.Path(data[0])
-        wnid = cast(Match[str], self._TRAIN_IMAGE_NAME_PATTERN.match(path.name))["wnid"]
-        label = Label.from_category(self._wnid_to_category[wnid], categories=self._categories)
-        return (label, wnid), data
-
-    def _prepare_test_data(self, data: Tuple[str, BinaryIO]) -> Tuple[None, Tuple[str, BinaryIO]]:
-        return None, data
-
-    def _classifiy_devkit(self, data: Tuple[str, BinaryIO]) -> Optional[int]:
-        return {
-            "meta.mat": ImageNetDemux.META,
-            "ILSVRC2012_validation_ground_truth.txt": ImageNetDemux.LABEL,
-        }.get(pathlib.Path(data[0]).name)
-
-    _VAL_TEST_IMAGE_NAME_PATTERN = re.compile(r"ILSVRC2012_(val|test)_(?P<id>\d{8})[.]JPEG")
-
-    def _val_test_image_key(self, path: pathlib.Path) -> int:
-        return int(self._VAL_TEST_IMAGE_NAME_PATTERN.match(path.name)["id"])  # type: ignore[index]
-
-    def _prepare_val_data(
-        self, data: Tuple[Tuple[int, str], Tuple[str, BinaryIO]]
-    ) -> Tuple[Tuple[Label, str], Tuple[str, BinaryIO]]:
-        label_data, image_data = data
-        _, wnid = label_data
-        label = Label.from_category(self._wnid_to_category[wnid], categories=self._categories)
-        return (label, wnid), image_data
-
-    def _prepare_sample(
-        self,
-        data: Tuple[Optional[Tuple[Label, str]], Tuple[str, BinaryIO]],
-    ) -> Dict[str, Any]:
-        label_data, (path, buffer) = data
-
-        return dict(
-            dict(zip(("label", "wnid"), label_data if label_data else (None, None))),
-            path=path,
-            image=EncodedImage.from_file(buffer),
-        )
-
-    def _datapipe(self, resource_dps: List[IterDataPipe]) -> IterDataPipe[Dict[str, Any]]:
-        if self._split in {"train", "test"}:
-            dp = resource_dps[0]
-
-            # the train archive is a tar of tars
-            if self._split == "train":
-                dp = TarArchiveLoader(dp)
-
-            dp = hint_shuffling(dp)
-            dp = hint_sharding(dp)
-            dp = Mapper(dp, self._prepare_train_data if self._split == "train" else self._prepare_test_data)
-        else:  # config.split == "val":
-            images_dp, devkit_dp = resource_dps
-
-            meta_dp, label_dp = Demultiplexer(
-                devkit_dp, 2, self._classifiy_devkit, drop_none=True, buffer_size=INFINITE_BUFFER_SIZE
-            )
-
-            # We cannot use self._wnids here, since we use a different order than the dataset
-            meta_dp = CategoryAndWordNetIDExtractor(meta_dp)
-            wnid_dp = Mapper(meta_dp, getitem(1))
-            wnid_dp = Enumerator(wnid_dp, 1)
-            wnid_map = IterToMapConverter(wnid_dp)
-
-            label_dp = LineReader(label_dp, decode=True, return_path=False)
-            label_dp = Mapper(label_dp, int)
-            label_dp = Mapper(label_dp, wnid_map.__getitem__)
-            label_dp: IterDataPipe[Tuple[int, str]] = Enumerator(label_dp, 1)
-            label_dp = hint_shuffling(label_dp)
-            label_dp = hint_sharding(label_dp)
-
-            dp = IterKeyZipper(
-                label_dp,
-                images_dp,
-                key_fn=getitem(0),
-                ref_key_fn=path_accessor(self._val_test_image_key),
-                buffer_size=INFINITE_BUFFER_SIZE,
-            )
-            dp = Mapper(dp, self._prepare_val_data)
-
-        return Mapper(dp, self._prepare_sample)
-
-    def __len__(self) -> int:
-        return {
-            "train": 1_281_167,
-            "val": 50_000,
-            "test": 100_000,
-        }[self._split]
-
-    def _filter_meta(self, data: Tuple[str, Any]) -> bool:
-        return self._classifiy_devkit(data) == ImageNetDemux.META
-
-    def _generate_categories(self) -> List[Tuple[str, ...]]:
-        self._split = "val"
-        resources = self._resources()
-
-        devkit_dp = resources[1].load(self._root)
-        meta_dp = Filter(devkit_dp, self._filter_meta)
-        meta_dp = CategoryAndWordNetIDExtractor(meta_dp)
-
-        categories_and_wnids = cast(List[Tuple[str, ...]], list(meta_dp))
-        categories_and_wnids.sort(key=lambda category_and_wnid: category_and_wnid[1])
-        return categories_and_wnids

torchvision/prototype/datasets/_builtin/oxford-iiit-pet.categories

-Abyssinian
-American Bulldog
-American Pit Bull Terrier
-Basset Hound
-Beagle
-Bengal
-Birman
-Bombay
-Boxer
-British Shorthair
-Chihuahua
-Egyptian Mau
-English Cocker Spaniel
-English Setter
-German Shorthaired
-Great Pyrenees
-Havanese
-Japanese Chin
-Keeshond
-Leonberger
-Maine Coon
-Miniature Pinscher
-Newfoundland
-Persian
-Pomeranian
-Pug
-Ragdoll
-Russian Blue
-Saint Bernard
-Samoyed
-Scottish Terrier
-Shiba Inu
-Siamese
-Sphynx
-Staffordshire Bull Terrier
-Wheaten Terrier
-Yorkshire Terrier

torchvision/prototype/datasets/_builtin/oxford_iiit_pet.py

-import enum
-import pathlib
-from typing import Any, BinaryIO, Dict, List, Optional, Tuple, Union
-
-from torchdata.datapipes.iter import CSVDictParser, Demultiplexer, Filter, IterDataPipe, IterKeyZipper, Mapper
-from torchvision.prototype.datasets.utils import Dataset, EncodedImage, HttpResource, OnlineResource
-from torchvision.prototype.datasets.utils._internal import (
-    getitem,
-    hint_sharding,
-    hint_shuffling,
-    INFINITE_BUFFER_SIZE,
-    path_accessor,
-    path_comparator,
-    read_categories_file,
-)
-from torchvision.prototype.tv_tensors import Label
-
-from .._api import register_dataset, register_info
-
-
-NAME = "oxford-iiit-pet"
-
-
-class OxfordIIITPetDemux(enum.IntEnum):
-    SPLIT_AND_CLASSIFICATION = 0
-    SEGMENTATIONS = 1
-
-
-@register_info(NAME)
-def _info() -> Dict[str, Any]:
-    return dict(categories=read_categories_file(NAME))
-
-
-@register_dataset(NAME)
-class OxfordIIITPet(Dataset):
-    """Oxford IIIT Pet Dataset
-    homepage="https://www.robots.ox.ac.uk/~vgg/data/pets/",
-    """
-
-    def __init__(
-        self, root: Union[str, pathlib.Path], *, split: str = "trainval", skip_integrity_check: bool = False
-    ) -> None:
-        self._split = self._verify_str_arg(split, "split", {"trainval", "test"})
-        self._categories = _info()["categories"]
-        super().__init__(root, skip_integrity_check=skip_integrity_check)
-
-    def _resources(self) -> List[OnlineResource]:
-        images = HttpResource(
-            "https://www.robots.ox.ac.uk/~vgg/data/pets/data/images.tar.gz",
-            sha256="67195c5e1c01f1ab5f9b6a5d22b8c27a580d896ece458917e61d459337fa318d",
-            preprocess="decompress",
-        )
-        anns = HttpResource(
-            "https://www.robots.ox.ac.uk/~vgg/data/pets/data/annotations.tar.gz",
-            sha256="52425fb6de5c424942b7626b428656fcbd798db970a937df61750c0f1d358e91",
-            preprocess="decompress",
-        )
-        return [images, anns]
-
-    def _classify_anns(self, data: Tuple[str, Any]) -> Optional[int]:
-        return {
-            "annotations": OxfordIIITPetDemux.SPLIT_AND_CLASSIFICATION,
-            "trimaps": OxfordIIITPetDemux.SEGMENTATIONS,
-        }.get(pathlib.Path(data[0]).parent.name)
-
-    def _filter_images(self, data: Tuple[str, Any]) -> bool:
-        return pathlib.Path(data[0]).suffix == ".jpg"
-
-    def _filter_segmentations(self, data: Tuple[str, Any]) -> bool:
-        return not pathlib.Path(data[0]).name.startswith(".")
-
-    def _prepare_sample(
-        self, data: Tuple[Tuple[Dict[str, str], Tuple[str, BinaryIO]], Tuple[str, BinaryIO]]
-    ) -> Dict[str, Any]:
-        ann_data, image_data = data
-        classification_data, segmentation_data = ann_data
-        segmentation_path, segmentation_buffer = segmentation_data
-        image_path, image_buffer = image_data
-
-        return dict(
-            label=Label(int(classification_data["label"]) - 1, categories=self._categories),
-            species="cat" if classification_data["species"] == "1" else "dog",
-            segmentation_path=segmentation_path,
-            segmentation=EncodedImage.from_file(segmentation_buffer),
-            image_path=image_path,
-            image=EncodedImage.from_file(image_buffer),
-        )
-
-    def _datapipe(self, resource_dps: List[IterDataPipe]) -> IterDataPipe[Dict[str, Any]]:
-        images_dp, anns_dp = resource_dps
-
-        images_dp = Filter(images_dp, self._filter_images)
-
-        split_and_classification_dp, segmentations_dp = Demultiplexer(
-            anns_dp,
-            2,
-            self._classify_anns,
-            drop_none=True,
-            buffer_size=INFINITE_BUFFER_SIZE,
-        )
-
-        split_and_classification_dp = Filter(split_and_classification_dp, path_comparator("name", f"{self._split}.txt"))
-        split_and_classification_dp = CSVDictParser(
-            split_and_classification_dp, fieldnames=("image_id", "label", "species"), delimiter=" "
-        )
-        split_and_classification_dp = hint_shuffling(split_and_classification_dp)
-        split_and_classification_dp = hint_sharding(split_and_classification_dp)
-
-        segmentations_dp = Filter(segmentations_dp, self._filter_segmentations)
-
-        anns_dp = IterKeyZipper(
-            split_and_classification_dp,
-            segmentations_dp,
-            key_fn=getitem("image_id"),
-            ref_key_fn=path_accessor("stem"),
-            buffer_size=INFINITE_BUFFER_SIZE,
-        )
-
-        dp = IterKeyZipper(
-            anns_dp,
-            images_dp,
-            key_fn=getitem(0, "image_id"),
-            ref_key_fn=path_accessor("stem"),
-            buffer_size=INFINITE_BUFFER_SIZE,
-        )
-        return Mapper(dp, self._prepare_sample)
-
-    def _filter_split_and_classification_anns(self, data: Tuple[str, Any]) -> bool:
-        return self._classify_anns(data) == OxfordIIITPetDemux.SPLIT_AND_CLASSIFICATION
-
-    def _generate_categories(self) -> List[str]:
-        resources = self._resources()
-
-        dp = resources[1].load(self._root)
-        dp = Filter(dp, self._filter_split_and_classification_anns)
-        dp = Filter(dp, path_comparator("name", "trainval.txt"))
-        dp = CSVDictParser(dp, fieldnames=("image_id", "label"), delimiter=" ")
-
-        raw_categories_and_labels = {(data["image_id"].rsplit("_", 1)[0], data["label"]) for data in dp}
-        raw_categories, _ = zip(
-            *sorted(raw_categories_and_labels, key=lambda raw_category_and_label: int(raw_category_and_label[1]))
-        )
-        return [" ".join(part.title() for part in raw_category.split("_")) for raw_category in raw_categories]
-
-    def __len__(self) -> int:
-        return 3_680 if self._split == "trainval" else 3_669

torchvision/prototype/datasets/_builtin/sbd.categories

-aeroplane
-bicycle
-bird
-boat
-bottle
-bus
-car
-cat
-chair
-cow
-diningtable
-dog
-horse
-motorbike
-person
-pottedplant
-sheep
-sofa
-train
-tvmonitor