Unverified Commit 58f313bb authored by Philip Meier's avatar Philip Meier Committed by GitHub
Browse files

add prototype imagenet dataset (#4640)

* add prototype imagenet dataset

* add missing checksums

* fix mypy

* add human readable categories

* cleanup

* sort categories ascending based on wnid

* remove accidentally added file

* cleanup category file generation

* fix mypy
parent 588c698d
......@@ -2,6 +2,7 @@ from .caltech import Caltech101, Caltech256
from .celeba import CelebA
from .cifar import Cifar10, Cifar100
from .coco import Coco
from .imagenet import ImageNet
from .mnist import MNIST, FashionMNIST, KMNIST, EMNIST, QMNIST
from .sbd import SBD
from .voc import VOC
tench,n01440764
goldfish,n01443537
great white shark,n01484850
tiger shark,n01491361
hammerhead,n01494475
electric ray,n01496331
stingray,n01498041
cock,n01514668
hen,n01514859
ostrich,n01518878
brambling,n01530575
goldfinch,n01531178
house finch,n01532829
junco,n01534433
indigo bunting,n01537544
robin,n01558993
bulbul,n01560419
jay,n01580077
magpie,n01582220
chickadee,n01592084
water ouzel,n01601694
kite,n01608432
bald eagle,n01614925
vulture,n01616318
great grey owl,n01622779
European fire salamander,n01629819
common newt,n01630670
eft,n01631663
spotted salamander,n01632458
axolotl,n01632777
bullfrog,n01641577
tree frog,n01644373
tailed frog,n01644900
loggerhead,n01664065
leatherback turtle,n01665541
mud turtle,n01667114
terrapin,n01667778
box turtle,n01669191
banded gecko,n01675722
common iguana,n01677366
American chameleon,n01682714
whiptail,n01685808
agama,n01687978
frilled lizard,n01688243
alligator lizard,n01689811
Gila monster,n01692333
green lizard,n01693334
African chameleon,n01694178
Komodo dragon,n01695060
African crocodile,n01697457
American alligator,n01698640
triceratops,n01704323
thunder snake,n01728572
ringneck snake,n01728920
hognose snake,n01729322
green snake,n01729977
king snake,n01734418
garter snake,n01735189
water snake,n01737021
vine snake,n01739381
night snake,n01740131
boa constrictor,n01742172
rock python,n01744401
Indian cobra,n01748264
green mamba,n01749939
sea snake,n01751748
horned viper,n01753488
diamondback,n01755581
sidewinder,n01756291
trilobite,n01768244
harvestman,n01770081
scorpion,n01770393
black and gold garden spider,n01773157
barn spider,n01773549
garden spider,n01773797
black widow,n01774384
tarantula,n01774750
wolf spider,n01775062
tick,n01776313
centipede,n01784675
black grouse,n01795545
ptarmigan,n01796340
ruffed grouse,n01797886
prairie chicken,n01798484
peacock,n01806143
quail,n01806567
partridge,n01807496
African grey,n01817953
macaw,n01818515
sulphur-crested cockatoo,n01819313
lorikeet,n01820546
coucal,n01824575
bee eater,n01828970
hornbill,n01829413
hummingbird,n01833805
jacamar,n01843065
toucan,n01843383
drake,n01847000
red-breasted merganser,n01855032
goose,n01855672
black swan,n01860187
tusker,n01871265
echidna,n01872401
platypus,n01873310
wallaby,n01877812
koala,n01882714
wombat,n01883070
jellyfish,n01910747
sea anemone,n01914609
brain coral,n01917289
flatworm,n01924916
nematode,n01930112
conch,n01943899
snail,n01944390
slug,n01945685
sea slug,n01950731
chiton,n01955084
chambered nautilus,n01968897
Dungeness crab,n01978287
rock crab,n01978455
fiddler crab,n01980166
king crab,n01981276
American lobster,n01983481
spiny lobster,n01984695
crayfish,n01985128
hermit crab,n01986214
isopod,n01990800
white stork,n02002556
black stork,n02002724
spoonbill,n02006656
flamingo,n02007558
little blue heron,n02009229
American egret,n02009912
bittern,n02011460
crane,n02012849
limpkin,n02013706
European gallinule,n02017213
American coot,n02018207
bustard,n02018795
ruddy turnstone,n02025239
red-backed sandpiper,n02027492
redshank,n02028035
dowitcher,n02033041
oystercatcher,n02037110
pelican,n02051845
king penguin,n02056570
albatross,n02058221
grey whale,n02066245
killer whale,n02071294
dugong,n02074367
sea lion,n02077923
Chihuahua,n02085620
Japanese spaniel,n02085782
Maltese dog,n02085936
Pekinese,n02086079
Shih-Tzu,n02086240
Blenheim spaniel,n02086646
papillon,n02086910
toy terrier,n02087046
Rhodesian ridgeback,n02087394
Afghan hound,n02088094
basset,n02088238
beagle,n02088364
bloodhound,n02088466
bluetick,n02088632
black-and-tan coonhound,n02089078
Walker hound,n02089867
English foxhound,n02089973
redbone,n02090379
borzoi,n02090622
Irish wolfhound,n02090721
Italian greyhound,n02091032
whippet,n02091134
Ibizan hound,n02091244
Norwegian elkhound,n02091467
otterhound,n02091635
Saluki,n02091831
Scottish deerhound,n02092002
Weimaraner,n02092339
Staffordshire bullterrier,n02093256
American Staffordshire terrier,n02093428
Bedlington terrier,n02093647
Border terrier,n02093754
Kerry blue terrier,n02093859
Irish terrier,n02093991
Norfolk terrier,n02094114
Norwich terrier,n02094258
Yorkshire terrier,n02094433
wire-haired fox terrier,n02095314
Lakeland terrier,n02095570
Sealyham terrier,n02095889
Airedale,n02096051
cairn,n02096177
Australian terrier,n02096294
Dandie Dinmont,n02096437
Boston bull,n02096585
miniature schnauzer,n02097047
giant schnauzer,n02097130
standard schnauzer,n02097209
Scotch terrier,n02097298
Tibetan terrier,n02097474
silky terrier,n02097658
soft-coated wheaten terrier,n02098105
West Highland white terrier,n02098286
Lhasa,n02098413
flat-coated retriever,n02099267
curly-coated retriever,n02099429
golden retriever,n02099601
Labrador retriever,n02099712
Chesapeake Bay retriever,n02099849
German short-haired pointer,n02100236
vizsla,n02100583
English setter,n02100735
Irish setter,n02100877
Gordon setter,n02101006
Brittany spaniel,n02101388
clumber,n02101556
English springer,n02102040
Welsh springer spaniel,n02102177
cocker spaniel,n02102318
Sussex spaniel,n02102480
Irish water spaniel,n02102973
kuvasz,n02104029
schipperke,n02104365
groenendael,n02105056
malinois,n02105162
briard,n02105251
kelpie,n02105412
komondor,n02105505
Old English sheepdog,n02105641
Shetland sheepdog,n02105855
collie,n02106030
Border collie,n02106166
Bouvier des Flandres,n02106382
Rottweiler,n02106550
German shepherd,n02106662
Doberman,n02107142
miniature pinscher,n02107312
Greater Swiss Mountain dog,n02107574
Bernese mountain dog,n02107683
Appenzeller,n02107908
EntleBucher,n02108000
boxer,n02108089
bull mastiff,n02108422
Tibetan mastiff,n02108551
French bulldog,n02108915
Great Dane,n02109047
Saint Bernard,n02109525
Eskimo dog,n02109961
malamute,n02110063
Siberian husky,n02110185
dalmatian,n02110341
affenpinscher,n02110627
basenji,n02110806
pug,n02110958
Leonberg,n02111129
Newfoundland,n02111277
Great Pyrenees,n02111500
Samoyed,n02111889
Pomeranian,n02112018
chow,n02112137
keeshond,n02112350
Brabancon griffon,n02112706
Pembroke,n02113023
Cardigan,n02113186
toy poodle,n02113624
miniature poodle,n02113712
standard poodle,n02113799
Mexican hairless,n02113978
timber wolf,n02114367
white wolf,n02114548
red wolf,n02114712
coyote,n02114855
dingo,n02115641
dhole,n02115913
African hunting dog,n02116738
hyena,n02117135
red fox,n02119022
kit fox,n02119789
Arctic fox,n02120079
grey fox,n02120505
tabby,n02123045
tiger cat,n02123159
Persian cat,n02123394
Siamese cat,n02123597
Egyptian cat,n02124075
cougar,n02125311
lynx,n02127052
leopard,n02128385
snow leopard,n02128757
jaguar,n02128925
lion,n02129165
tiger,n02129604
cheetah,n02130308
brown bear,n02132136
American black bear,n02133161
ice bear,n02134084
sloth bear,n02134418
mongoose,n02137549
meerkat,n02138441
tiger beetle,n02165105
ladybug,n02165456
ground beetle,n02167151
long-horned beetle,n02168699
leaf beetle,n02169497
dung beetle,n02172182
rhinoceros beetle,n02174001
weevil,n02177972
fly,n02190166
bee,n02206856
ant,n02219486
grasshopper,n02226429
cricket,n02229544
walking stick,n02231487
cockroach,n02233338
mantis,n02236044
cicada,n02256656
leafhopper,n02259212
lacewing,n02264363
dragonfly,n02268443
damselfly,n02268853
admiral,n02276258
ringlet,n02277742
monarch,n02279972
cabbage butterfly,n02280649
sulphur butterfly,n02281406
lycaenid,n02281787
starfish,n02317335
sea urchin,n02319095
sea cucumber,n02321529
wood rabbit,n02325366
hare,n02326432
Angora,n02328150
hamster,n02342885
porcupine,n02346627
fox squirrel,n02356798
marmot,n02361337
beaver,n02363005
guinea pig,n02364673
sorrel,n02389026
zebra,n02391049
hog,n02395406
wild boar,n02396427
warthog,n02397096
hippopotamus,n02398521
ox,n02403003
water buffalo,n02408429
bison,n02410509
ram,n02412080
bighorn,n02415577
ibex,n02417914
hartebeest,n02422106
impala,n02422699
gazelle,n02423022
Arabian camel,n02437312
llama,n02437616
weasel,n02441942
mink,n02442845
polecat,n02443114
black-footed ferret,n02443484
otter,n02444819
skunk,n02445715
badger,n02447366
armadillo,n02454379
three-toed sloth,n02457408
orangutan,n02480495
gorilla,n02480855
chimpanzee,n02481823
gibbon,n02483362
siamang,n02483708
guenon,n02484975
patas,n02486261
baboon,n02486410
macaque,n02487347
langur,n02488291
colobus,n02488702
proboscis monkey,n02489166
marmoset,n02490219
capuchin,n02492035
howler monkey,n02492660
titi,n02493509
spider monkey,n02493793
squirrel monkey,n02494079
Madagascar cat,n02497673
indri,n02500267
Indian elephant,n02504013
African elephant,n02504458
lesser panda,n02509815
giant panda,n02510455
barracouta,n02514041
eel,n02526121
coho,n02536864
rock beauty,n02606052
anemone fish,n02607072
sturgeon,n02640242
gar,n02641379
lionfish,n02643566
puffer,n02655020
abacus,n02666196
abaya,n02667093
academic gown,n02669723
accordion,n02672831
acoustic guitar,n02676566
aircraft carrier,n02687172
airliner,n02690373
airship,n02692877
altar,n02699494
ambulance,n02701002
amphibian,n02704792
analog clock,n02708093
apiary,n02727426
apron,n02730930
ashcan,n02747177
assault rifle,n02749479
backpack,n02769748
bakery,n02776631
balance beam,n02777292
balloon,n02782093
ballpoint,n02783161
Band Aid,n02786058
banjo,n02787622
bannister,n02788148
barbell,n02790996
barber chair,n02791124
barbershop,n02791270
barn,n02793495
barometer,n02794156
barrel,n02795169
barrow,n02797295
baseball,n02799071
basketball,n02802426
bassinet,n02804414
bassoon,n02804610
bathing cap,n02807133
bath towel,n02808304
bathtub,n02808440
beach wagon,n02814533
beacon,n02814860
beaker,n02815834
bearskin,n02817516
beer bottle,n02823428
beer glass,n02823750
bell cote,n02825657
bib,n02834397
bicycle-built-for-two,n02835271
bikini,n02837789
binder,n02840245
binoculars,n02841315
birdhouse,n02843684
boathouse,n02859443
bobsled,n02860847
bolo tie,n02865351
bonnet,n02869837
bookcase,n02870880
bookshop,n02871525
bottlecap,n02877765
bow,n02879718
bow tie,n02883205
brass,n02892201
brassiere,n02892767
breakwater,n02894605
breastplate,n02895154
broom,n02906734
bucket,n02909870
buckle,n02910353
bulletproof vest,n02916936
bullet train,n02917067
butcher shop,n02927161
cab,n02930766
caldron,n02939185
candle,n02948072
cannon,n02950826
canoe,n02951358
can opener,n02951585
cardigan,n02963159
car mirror,n02965783
carousel,n02966193
carpenter's kit,n02966687
carton,n02971356
car wheel,n02974003
cash machine,n02977058
cassette,n02978881
cassette player,n02979186
castle,n02980441
catamaran,n02981792
CD player,n02988304
cello,n02992211
cellular telephone,n02992529
chain,n02999410
chainlink fence,n03000134
chain mail,n03000247
chain saw,n03000684
chest,n03014705
chiffonier,n03016953
chime,n03017168
china cabinet,n03018349
Christmas stocking,n03026506
church,n03028079
cinema,n03032252
cleaver,n03041632
cliff dwelling,n03042490
cloak,n03045698
clog,n03047690
cocktail shaker,n03062245
coffee mug,n03063599
coffeepot,n03063689
coil,n03065424
combination lock,n03075370
computer keyboard,n03085013
confectionery,n03089624
container ship,n03095699
convertible,n03100240
corkscrew,n03109150
cornet,n03110669
cowboy boot,n03124043
cowboy hat,n03124170
cradle,n03125729
construction crane,n03126707
crash helmet,n03127747
crate,n03127925
crib,n03131574
Crock Pot,n03133878
croquet ball,n03134739
crutch,n03141823
cuirass,n03146219
dam,n03160309
desk,n03179701
desktop computer,n03180011
dial telephone,n03187595
diaper,n03188531
digital clock,n03196217
digital watch,n03197337
dining table,n03201208
dishrag,n03207743
dishwasher,n03207941
disk brake,n03208938
dock,n03216828
dogsled,n03218198
dome,n03220513
doormat,n03223299
drilling platform,n03240683
drum,n03249569
drumstick,n03250847
dumbbell,n03255030
Dutch oven,n03259280
electric fan,n03271574
electric guitar,n03272010
electric locomotive,n03272562
entertainment center,n03290653
envelope,n03291819
espresso maker,n03297495
face powder,n03314780
feather boa,n03325584
file,n03337140
fireboat,n03344393
fire engine,n03345487
fire screen,n03347037
flagpole,n03355925
flute,n03372029
folding chair,n03376595
football helmet,n03379051
forklift,n03384352
fountain,n03388043
fountain pen,n03388183
four-poster,n03388549
freight car,n03393912
French horn,n03394916
frying pan,n03400231
fur coat,n03404251
garbage truck,n03417042
gasmask,n03424325
gas pump,n03425413
goblet,n03443371
go-kart,n03444034
golf ball,n03445777
golfcart,n03445924
gondola,n03447447
gong,n03447721
gown,n03450230
grand piano,n03452741
greenhouse,n03457902
grille,n03459775
grocery store,n03461385
guillotine,n03467068
hair slide,n03476684
hair spray,n03476991
half track,n03478589
hammer,n03481172
hamper,n03482405
hand blower,n03483316
hand-held computer,n03485407
handkerchief,n03485794
hard disc,n03492542
harmonica,n03494278
harp,n03495258
harvester,n03496892
hatchet,n03498962
holster,n03527444
home theater,n03529860
honeycomb,n03530642
hook,n03532672
hoopskirt,n03534580
horizontal bar,n03535780
horse cart,n03538406
hourglass,n03544143
iPod,n03584254
iron,n03584829
jack-o'-lantern,n03590841
jean,n03594734
jeep,n03594945
jersey,n03595614
jigsaw puzzle,n03598930
jinrikisha,n03599486
joystick,n03602883
kimono,n03617480
knee pad,n03623198
knot,n03627232
lab coat,n03630383
ladle,n03633091
lampshade,n03637318
laptop,n03642806
lawn mower,n03649909
lens cap,n03657121
letter opener,n03658185
library,n03661043
lifeboat,n03662601
lighter,n03666591
limousine,n03670208
liner,n03673027
lipstick,n03676483
Loafer,n03680355
lotion,n03690938
loudspeaker,n03691459
loupe,n03692522
lumbermill,n03697007
magnetic compass,n03706229
mailbag,n03709823
mailbox,n03710193
maillot,n03710637
tank suit,n03710721
manhole cover,n03717622
maraca,n03720891
marimba,n03721384
mask,n03724870
matchstick,n03729826
maypole,n03733131
maze,n03733281
measuring cup,n03733805
medicine chest,n03742115
megalith,n03743016
microphone,n03759954
microwave,n03761084
military uniform,n03763968
milk can,n03764736
minibus,n03769881
miniskirt,n03770439
minivan,n03770679
missile,n03773504
mitten,n03775071
mixing bowl,n03775546
mobile home,n03776460
Model T,n03777568
modem,n03777754
monastery,n03781244
monitor,n03782006
moped,n03785016
mortar,n03786901
mortarboard,n03787032
mosque,n03788195
mosquito net,n03788365
motor scooter,n03791053
mountain bike,n03792782
mountain tent,n03792972
mouse,n03793489
mousetrap,n03794056
moving van,n03796401
muzzle,n03803284
nail,n03804744
neck brace,n03814639
necklace,n03814906
nipple,n03825788
notebook,n03832673
obelisk,n03837869
oboe,n03838899
ocarina,n03840681
odometer,n03841143
oil filter,n03843555
organ,n03854065
oscilloscope,n03857828
overskirt,n03866082
oxcart,n03868242
oxygen mask,n03868863
packet,n03871628
paddle,n03873416
paddlewheel,n03874293
padlock,n03874599
paintbrush,n03876231
pajama,n03877472
palace,n03877845
panpipe,n03884397
paper towel,n03887697
parachute,n03888257
parallel bars,n03888605
park bench,n03891251
parking meter,n03891332
passenger car,n03895866
patio,n03899768
pay-phone,n03902125
pedestal,n03903868
pencil box,n03908618
pencil sharpener,n03908714
perfume,n03916031
Petri dish,n03920288
photocopier,n03924679
pick,n03929660
pickelhaube,n03929855
picket fence,n03930313
pickup,n03930630
pier,n03933933
piggy bank,n03935335
pill bottle,n03937543
pillow,n03938244
ping-pong ball,n03942813
pinwheel,n03944341
pirate,n03947888
pitcher,n03950228
plane,n03954731
planetarium,n03956157
plastic bag,n03958227
plate rack,n03961711
plow,n03967562
plunger,n03970156
Polaroid camera,n03976467
pole,n03976657
police van,n03977966
poncho,n03980874
pool table,n03982430
pop bottle,n03983396
pot,n03991062
potter's wheel,n03992509
power drill,n03995372
prayer rug,n03998194
printer,n04004767
prison,n04005630
projectile,n04008634
projector,n04009552
puck,n04019541
punching bag,n04023962
purse,n04026417
quill,n04033901
quilt,n04033995
racer,n04037443
racket,n04039381
radiator,n04040759
radio,n04041544
radio telescope,n04044716
rain barrel,n04049303
recreational vehicle,n04065272
reel,n04067472
reflex camera,n04069434
refrigerator,n04070727
remote control,n04074963
restaurant,n04081281
revolver,n04086273
rifle,n04090263
rocking chair,n04099969
rotisserie,n04111531
rubber eraser,n04116512
rugby ball,n04118538
rule,n04118776
running shoe,n04120489
safe,n04125021
safety pin,n04127249
saltshaker,n04131690
sandal,n04133789
sarong,n04136333
sax,n04141076
scabbard,n04141327
scale,n04141975
school bus,n04146614
schooner,n04147183
scoreboard,n04149813
screen,n04152593
screw,n04153751
screwdriver,n04154565
seat belt,n04162706
sewing machine,n04179913
shield,n04192698
shoe shop,n04200800
shoji,n04201297
shopping basket,n04204238
shopping cart,n04204347
shovel,n04208210
shower cap,n04209133
shower curtain,n04209239
ski,n04228054
ski mask,n04229816
sleeping bag,n04235860
slide rule,n04238763
sliding door,n04239074
slot,n04243546
snorkel,n04251144
snowmobile,n04252077
snowplow,n04252225
soap dispenser,n04254120
soccer ball,n04254680
sock,n04254777
solar dish,n04258138
sombrero,n04259630
soup bowl,n04263257
space bar,n04264628
space heater,n04265275
space shuttle,n04266014
spatula,n04270147
speedboat,n04273569
spider web,n04275548
spindle,n04277352
sports car,n04285008
spotlight,n04286575
stage,n04296562
steam locomotive,n04310018
steel arch bridge,n04311004
steel drum,n04311174
stethoscope,n04317175
stole,n04325704
stone wall,n04326547
stopwatch,n04328186
stove,n04330267
strainer,n04332243
streetcar,n04335435
stretcher,n04336792
studio couch,n04344873
stupa,n04346328
submarine,n04347754
suit,n04350905
sundial,n04355338
sunglass,n04355933
sunglasses,n04356056
sunscreen,n04357314
suspension bridge,n04366367
swab,n04367480
sweatshirt,n04370456
swimming trunks,n04371430
swing,n04371774
switch,n04372370
syringe,n04376876
table lamp,n04380533
tank,n04389033
tape player,n04392985
teapot,n04398044
teddy,n04399382
television,n04404412
tennis ball,n04409515
thatch,n04417672
theater curtain,n04418357
thimble,n04423845
thresher,n04428191
throne,n04429376
tile roof,n04435653
toaster,n04442312
tobacco shop,n04443257
toilet seat,n04447861
torch,n04456115
totem pole,n04458633
tow truck,n04461696
toyshop,n04462240
tractor,n04465501
trailer truck,n04467665
tray,n04476259
trench coat,n04479046
tricycle,n04482393
trimaran,n04483307
tripod,n04485082
triumphal arch,n04486054
trolleybus,n04487081
trombone,n04487394
tub,n04493381
turnstile,n04501370
typewriter keyboard,n04505470
umbrella,n04507155
unicycle,n04509417
upright,n04515003
vacuum,n04517823
vase,n04522168
vault,n04523525
velvet,n04525038
vending machine,n04525305
vestment,n04532106
viaduct,n04532670
violin,n04536866
volleyball,n04540053
waffle iron,n04542943
wall clock,n04548280
wallet,n04548362
wardrobe,n04550184
warplane,n04552348
washbasin,n04553703
washer,n04554684
water bottle,n04557648
water jug,n04560804
water tower,n04562935
whiskey jug,n04579145
whistle,n04579432
wig,n04584207
window screen,n04589890
window shade,n04590129
Windsor tie,n04591157
wine bottle,n04591713
wing,n04592741
wok,n04596742
wooden spoon,n04597913
wool,n04599235
worm fence,n04604644
wreck,n04606251
yawl,n04612504
yurt,n04613696
web site,n06359193
comic book,n06596364
crossword puzzle,n06785654
street sign,n06794110
traffic light,n06874185
book jacket,n07248320
menu,n07565083
plate,n07579787
guacamole,n07583066
consomme,n07584110
hot pot,n07590611
trifle,n07613480
ice cream,n07614500
ice lolly,n07615774
French loaf,n07684084
bagel,n07693725
pretzel,n07695742
cheeseburger,n07697313
hotdog,n07697537
mashed potato,n07711569
head cabbage,n07714571
broccoli,n07714990
cauliflower,n07715103
zucchini,n07716358
spaghetti squash,n07716906
acorn squash,n07717410
butternut squash,n07717556
cucumber,n07718472
artichoke,n07718747
bell pepper,n07720875
cardoon,n07730033
mushroom,n07734744
Granny Smith,n07742313
strawberry,n07745940
orange,n07747607
lemon,n07749582
fig,n07753113
pineapple,n07753275
banana,n07753592
jackfruit,n07754684
custard apple,n07760859
pomegranate,n07768694
hay,n07802026
carbonara,n07831146
chocolate sauce,n07836838
dough,n07860988
meat loaf,n07871810
pizza,n07873807
potpie,n07875152
burrito,n07880968
red wine,n07892512
espresso,n07920052
cup,n07930864
eggnog,n07932039
alp,n09193705
bubble,n09229709
cliff,n09246464
coral reef,n09256479
geyser,n09288635
lakeside,n09332890
promontory,n09399592
sandbar,n09421951
seashore,n09428293
valley,n09468604
volcano,n09472597
ballplayer,n09835506
groom,n10148035
scuba diver,n10565667
rapeseed,n11879895
daisy,n11939491
yellow lady's slipper,n12057211
corn,n12144580
acorn,n12267677
hip,n12620546
buckeye,n12768682
coral fungus,n12985857
agaric,n12998815
gyromitra,n13037406
stinkhorn,n13040303
earthstar,n13044778
hen-of-the-woods,n13052670
bolete,n13054560
ear,n13133613
toilet tissue,n15075141
import io
import pathlib
import re
from typing import Any, Callable, Dict, List, Optional, Tuple
import torch
from torchdata.datapipes.iter import IterDataPipe, LineReader, KeyZipper, Mapper, TarArchiveReader, Filter, Shuffler
from torchvision.prototype.datasets.utils import (
Dataset,
DatasetConfig,
DatasetInfo,
HttpResource,
OnlineResource,
DatasetType,
)
from torchvision.prototype.datasets.utils._internal import (
INFINITE_BUFFER_SIZE,
BUILTIN_DIR,
path_comparator,
Enumerator,
getitem,
read_mat,
FrozenMapping,
)
class ImageNet(Dataset):
@property
def info(self) -> DatasetInfo:
name = "imagenet"
categories, wnids = zip(*DatasetInfo.read_categories_file(BUILTIN_DIR / f"{name}.categories"))
return DatasetInfo(
name,
type=DatasetType.IMAGE,
categories=categories,
homepage="https://www.image-net.org/",
valid_options=dict(split=("train", "val")),
extra=dict(
wnid_to_category=FrozenMapping(zip(wnids, categories)),
category_to_wnid=FrozenMapping(zip(categories, wnids)),
),
)
@property
def category_to_wnid(self) -> Dict[str, str]:
return self.info.extra.category_to_wnid
@property
def wnid_to_category(self) -> Dict[str, str]:
return self.info.extra.wnid_to_category
def resources(self, config: DatasetConfig) -> List[OnlineResource]:
if config.split == "train":
images = HttpResource(
"ILSVRC2012_img_train.tar",
sha256="b08200a27a8e34218a0e58fde36b0fe8f73bc377f4acea2d91602057c3ca45bb",
)
else: # config.split == "val"
images = HttpResource(
"ILSVRC2012_img_val.tar",
sha256="c7e06a6c0baccf06d8dbeb6577d71efff84673a5dbdd50633ab44f8ea0456ae0",
)
devkit = HttpResource(
"ILSVRC2012_devkit_t12.tar.gz",
sha256="b59243268c0d266621fd587d2018f69e906fb22875aca0e295b48cafaa927953",
)
return [images, devkit]
_TRAIN_IMAGE_NAME_PATTERN = re.compile(r"(?P<wnid>n\d{8})_\d+[.]JPEG")
def _collate_train_data(self, data: Tuple[str, io.IOBase]) -> Tuple[Tuple[int, str, str], Tuple[str, io.IOBase]]:
path = pathlib.Path(data[0])
wnid = self._TRAIN_IMAGE_NAME_PATTERN.match(path.name).group("wnid") # type: ignore[union-attr]
category = self.wnid_to_category[wnid]
label = self.categories.index(category)
return (label, category, wnid), data
_VAL_IMAGE_NAME_PATTERN = re.compile(r"ILSVRC2012_val_(?P<id>\d{8})[.]JPEG")
def _val_image_key(self, data: Tuple[str, Any]) -> int:
path = pathlib.Path(data[0])
return int(self._VAL_IMAGE_NAME_PATTERN.match(path.name).group("id")) # type: ignore[union-attr]
def _collate_val_data(
self, data: Tuple[Tuple[int, int], Tuple[str, io.IOBase]]
) -> Tuple[Tuple[int, str, str], Tuple[str, io.IOBase]]:
label_data, image_data = data
_, label = label_data
category = self.categories[label]
wnid = self.category_to_wnid[category]
return (label, category, wnid), image_data
def _collate_and_decode_sample(
self,
data: Tuple[Tuple[int, str, str], Tuple[str, io.IOBase]],
*,
decoder: Optional[Callable[[io.IOBase], torch.Tensor]],
) -> Dict[str, Any]:
ann_data, image_data = data
label, category, wnid = ann_data
path, buffer = image_data
return dict(
path=path,
image=decoder(buffer) if decoder else buffer,
label=torch.tensor(label),
category=category,
wnid=wnid,
)
def _make_datapipe(
self,
resource_dps: List[IterDataPipe],
*,
config: DatasetConfig,
decoder: Optional[Callable[[io.IOBase], torch.Tensor]],
) -> IterDataPipe[Dict[str, Any]]:
images_dp, devkit_dp = resource_dps
images_dp = TarArchiveReader(images_dp)
if config.split == "train":
# the train archive is a tar of tars
dp = TarArchiveReader(images_dp)
# dp = Shuffler(dp, buffer_size=INFINITE_BUFFER_SIZE)
dp = Mapper(dp, self._collate_train_data)
else:
devkit_dp = TarArchiveReader(devkit_dp)
devkit_dp = Filter(devkit_dp, path_comparator("name", "ILSVRC2012_validation_ground_truth.txt"))
devkit_dp = LineReader(devkit_dp, return_path=False)
devkit_dp = Mapper(devkit_dp, int)
devkit_dp = Enumerator(devkit_dp, 1)
devkit_dp = Shuffler(devkit_dp, buffer_size=INFINITE_BUFFER_SIZE)
dp = KeyZipper(
devkit_dp,
images_dp,
key_fn=getitem(0),
ref_key_fn=self._val_image_key,
buffer_size=INFINITE_BUFFER_SIZE,
)
dp = Mapper(dp, self._collate_val_data)
return Mapper(dp, self._collate_and_decode_sample, fn_kwargs=dict(decoder=decoder))
# Although the WordNet IDs (wnids) are unique, the corresponding categories are not. For example, both n02012849
# and n03126707 are labeled 'crane' while the first means the bird and the latter means the construction equipment
_WNID_MAP = {
"n03126707": "construction crane",
"n03710721": "tank suit",
}
def _generate_categories(self, root: pathlib.Path) -> List[Tuple[str, str]]:
resources = self.resources(self.default_config)
devkit_dp = resources[1].to_datapipe(root / self.name)
devkit_dp = TarArchiveReader(devkit_dp)
devkit_dp = Filter(devkit_dp, path_comparator("name", "meta.mat"))
meta = next(iter(devkit_dp))[1]
synsets = read_mat(meta, squeeze_me=True)["synsets"]
categories_and_wnids = [
(self._WNID_MAP.get(wnid, category.split(",", 1)[0]), wnid)
for _, wnid, category, _, num_children, *_ in synsets
# if num_children > 0, we are looking at a superclass that has no direct instance
if num_children == 0
]
categories_and_wnids.sort(key=lambda category_and_wnid: category_and_wnid[1])
return categories_and_wnids
import argparse
import csv
import sys
import unittest.mock
import warnings
with warnings.catch_warnings():
warnings.filterwarnings("ignore", message=r"The categories file .+? does not exist.", category=UserWarning)
from torchvision.prototype import datasets
from torchvision.prototype import datasets
from torchvision.prototype.datasets._api import find
from torchvision.prototype.datasets.utils._internal import BUILTIN_DIR
......@@ -16,21 +11,18 @@ def main(*names, force=False):
root = datasets.home()
for name in names:
file = BUILTIN_DIR / f"{name}.categories"
if file.exists() and not force:
path = BUILTIN_DIR / f"{name}.categories"
if path.exists() and not force:
continue
dataset = find(name)
try:
with unittest.mock.patch(
"torchvision.prototype.datasets.utils._dataset.DatasetInfo._read_categories_file", return_value=[]
):
categories = dataset._generate_categories(root)
categories = dataset._generate_categories(root)
except NotImplementedError:
continue
with open(file, "w") as fh:
fh.write("\n".join(categories) + "\n")
with open(path, "w", newline="") as file:
csv.writer(file).writerows(categories)
def parse_args(argv=None):
......
import abc
import csv
import enum
import io
import os
import pathlib
import textwrap
import warnings
from collections.abc import Mapping
from typing import (
Any,
Callable,
......@@ -14,8 +11,6 @@ from typing import (
Optional,
Sequence,
Union,
NoReturn,
Iterable,
Tuple,
)
......@@ -26,76 +21,17 @@ from torchvision.prototype.datasets.utils._internal import (
sequence_to_str,
)
from ._internal import FrozenBunch, make_repr
from ._resource import OnlineResource
def make_repr(name: str, items: Iterable[Tuple[str, Any]]):
def to_str(sep: str) -> str:
return sep.join([f"{key}={value}" for key, value in items])
prefix = f"{name}("
postfix = ")"
body = to_str(", ")
line_length = int(os.environ.get("COLUMNS", 80))
body_too_long = (len(prefix) + len(body) + len(postfix)) > line_length
multiline_body = len(str(body).splitlines()) > 1
if not (body_too_long or multiline_body):
return prefix + body + postfix
body = textwrap.indent(to_str(",\n"), " " * 2)
return f"{prefix}\n{body}\n{postfix}"
class DatasetType(enum.Enum):
RAW = enum.auto()
IMAGE = enum.auto()
class DatasetConfig(Mapping):
def __init__(self, *args, **kwargs):
data = dict(*args, **kwargs)
self.__dict__["__data__"] = data
self.__dict__["__final_hash__"] = hash(tuple(data.items()))
def __getitem__(self, name: str) -> Any:
return self.__dict__["__data__"][name]
def __iter__(self):
return iter(self.__dict__["__data__"].keys())
def __len__(self):
return len(self.__dict__["__data__"])
def __getattr__(self, name: str) -> Any:
try:
return self[name]
except KeyError as error:
raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'") from error
def __setitem__(self, key: Any, value: Any) -> NoReturn:
raise RuntimeError(f"'{type(self).__name__}' object is immutable")
def __setattr__(self, key: Any, value: Any) -> NoReturn:
raise RuntimeError(f"'{type(self).__name__}' object is immutable")
def __delitem__(self, key: Any) -> NoReturn:
raise RuntimeError(f"'{type(self).__name__}' object is immutable")
def __delattr__(self, item: Any) -> NoReturn:
raise RuntimeError(f"'{type(self).__name__}' object is immutable")
def __hash__(self) -> int:
return self.__dict__["__final_hash__"]
def __eq__(self, other: Any) -> bool:
if not isinstance(other, DatasetConfig):
return NotImplemented
return hash(self) == hash(other)
def __repr__(self) -> str:
return make_repr(type(self).__name__, self.items())
class DatasetConfig(FrozenBunch):
pass
class DatasetInfo:
......@@ -109,6 +45,7 @@ class DatasetInfo:
homepage: Optional[str] = None,
license: Optional[str] = None,
valid_options: Optional[Dict[str, Sequence]] = None,
extra: Optional[Dict[str, Any]] = None,
) -> None:
self.name = name.lower()
self.type = DatasetType[type.upper()] if isinstance(type, str) else type
......@@ -118,7 +55,8 @@ class DatasetInfo:
elif isinstance(categories, int):
categories = [str(label) for label in range(categories)]
elif isinstance(categories, (str, pathlib.Path)):
categories = self._read_categories_file(pathlib.Path(categories).expanduser().resolve())
path = pathlib.Path(categories).expanduser().resolve()
categories, *_ = zip(*self.read_categories_file(path))
self.categories = tuple(categories)
self.citation = citation
......@@ -137,16 +75,12 @@ class DatasetInfo:
)
self._valid_options: Dict[str, Sequence] = valid_options
@staticmethod
def _read_categories_file(path: pathlib.Path) -> List[str]:
if not path.exists() or not path.is_file():
warnings.warn(
f"The categories file {path} does not exist. Continuing without loaded categories.", UserWarning
)
return []
self.extra = FrozenBunch(extra or dict())
with open(path, "r") as file:
return [line.strip() for line in file]
@staticmethod
def read_categories_file(path: pathlib.Path) -> List[List[str]]:
with open(path, "r", newline="") as file:
return [row for row in csv.reader(file)]
@property
def default_config(self) -> DatasetConfig:
......@@ -231,5 +165,5 @@ class Dataset(abc.ABC):
resource_dps = [resource.to_datapipe(root) for resource in self.resources(config)]
return self._make_datapipe(resource_dps, config=config, decoder=decoder)
def _generate_categories(self, root: pathlib.Path) -> Sequence[str]:
def _generate_categories(self, root: pathlib.Path) -> Sequence[Union[str, Sequence[str]]]:
raise NotImplementedError
import collections.abc
import csv
import difflib
import enum
import gzip
import io
import lzma
import os
import os.path
import pathlib
from typing import Collection, Sequence, Callable, Union, Any, Tuple, TypeVar, Iterator, Dict, Optional
import textwrap
from collections.abc import Mapping
from typing import (
Collection,
Sequence,
Callable,
Union,
Any,
Tuple,
TypeVar,
Iterator,
Dict,
Optional,
NoReturn,
Iterable,
)
import numpy as np
import PIL.Image
......@@ -18,6 +35,10 @@ __all__ = [
"BUILTIN_DIR",
"sequence_to_str",
"add_suggestion",
"make_repr",
"FrozenMapping",
"FrozenBunch",
"create_categories_file",
"read_mat",
"image_buffer_from_array",
"SequenceIterator",
......@@ -62,6 +83,82 @@ def add_suggestion(
return f"{msg.strip()} {hint}"
def make_repr(name: str, items: Iterable[Tuple[str, Any]]):
def to_str(sep: str) -> str:
return sep.join([f"{key}={value}" for key, value in items])
prefix = f"{name}("
postfix = ")"
body = to_str(", ")
line_length = int(os.environ.get("COLUMNS", 80))
body_too_long = (len(prefix) + len(body) + len(postfix)) > line_length
multiline_body = len(str(body).splitlines()) > 1
if not (body_too_long or multiline_body):
return prefix + body + postfix
body = textwrap.indent(to_str(",\n"), " " * 2)
return f"{prefix}\n{body}\n{postfix}"
class FrozenMapping(Mapping):
def __init__(self, *args, **kwargs):
data = dict(*args, **kwargs)
self.__dict__["__data__"] = data
self.__dict__["__final_hash__"] = hash(tuple(data.items()))
def __getitem__(self, name: str) -> Any:
return self.__dict__["__data__"][name]
def __iter__(self):
return iter(self.__dict__["__data__"].keys())
def __len__(self):
return len(self.__dict__["__data__"])
def __setitem__(self, key: Any, value: Any) -> NoReturn:
raise RuntimeError(f"'{type(self).__name__}' object is immutable")
def __delitem__(self, key: Any) -> NoReturn:
raise RuntimeError(f"'{type(self).__name__}' object is immutable")
def __hash__(self) -> int:
return self.__dict__["__final_hash__"]
def __eq__(self, other: Any) -> bool:
if not isinstance(other, FrozenMapping):
return NotImplemented
return hash(self) == hash(other)
def __repr__(self):
return repr(self.__dict__["__data__"])
class FrozenBunch(FrozenMapping):
def __getattr__(self, name: str) -> Any:
try:
return self[name]
except KeyError as error:
raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'") from error
def __setattr__(self, key: Any, value: Any) -> NoReturn:
raise RuntimeError(f"'{type(self).__name__}' object is immutable")
def __delattr__(self, item: Any) -> NoReturn:
raise RuntimeError(f"'{type(self).__name__}' object is immutable")
def __repr__(self) -> str:
return make_repr(type(self).__name__, self.items())
def create_categories_file(
root: Union[str, pathlib.Path], name: str, categories: Sequence[Union[str, Sequence[str]]], **fmtparams: Any
) -> None:
with open(pathlib.Path(root) / f"{name}.categories", "w", newline="") as file:
csv.writer(file, **fmtparams).writerows(categories)
def read_mat(buffer: io.IOBase, **kwargs: Any) -> Any:
try:
import scipy.io as sio
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment