Add imagenet scripts. (#2185)

slim/BUILD

@@ -17,6 +17,28 @@ py_library(
     ],
 )
 
+sh_binary(
+    name = "download_and_convert_imagenet",
+    srcs = ["datasets/download_and_convert_imagenet.sh"],
+    data = [
+        "datasets/download_imagenet.sh",
+        "datasets/imagenet_2012_validation_synset_labels.txt",
+        "datasets/imagenet_lsvrc_2015_synsets.txt",
+        "datasets/imagenet_metadata.txt",
+        "datasets/preprocess_imagenet_validation_data.py",
+        "datasets/process_bounding_boxes.py",
+        ":build_imagenet_data",
+    ],
+)
+
+py_binary(
+    name = "build_imagenet_data",
+    srcs = ["datasets/build_imagenet_data.py"],
+    deps = [
+        "//tensorflow",
+    ],
+)
+
 py_library(
     name = "download_and_convert_cifar10",
     srcs = ["datasets/download_and_convert_cifar10.py"],

slim/README.md

@@ -163,7 +163,56 @@ dataset = flowers.get_split('validation', DATA_DIR)
 provider = slim.dataset_data_provider.DatasetDataProvider(dataset)
 [image, label] = provider.get(['image', 'label'])
 ```
+## An automated script for processing ImageNet data.
 
+Training a model with the ImageNet dataset is a common request. To facilitate
+working with the ImageNet dataset, we provide an automated script for
+downloading and processing the ImageNet dataset into the native TFRecord
+format.
+
+The TFRecord format consists of a set of sharded files where each entry is a serialized `tf.Example` proto. Each `tf.Example` proto contains the ImageNet image (JPEG encoded) as well as metadata such as label and bounding box information.
+
+We provide a single [script](datasets/download_and_preprocess_imagenet.sh) for
+downloading and converting ImageNet data to TFRecord format. Downloading and
+preprocessing the data may take several hours (up to half a day) depending on
+your network and computer speed. Please be patient.
+
+To begin, you will need to sign up for an account with [ImageNet]
+(http://image-net.org) to gain access to the data. Look for the sign up page,
+create an account and request an access key to download the data.
+
+After you have `USERNAME` and `PASSWORD`, you are ready to run our script. Make
+sure that your hard disk has at least 500 GB of free space for downloading and
+storing the data. Here we select `DATA_DIR=$HOME/imagenet-data` as such a
+location but feel free to edit accordingly.
+
+When you run the below script, please enter *USERNAME* and *PASSWORD* when
+prompted. This will occur at the very beginning. Once these values are entered,
+you will not need to interact with the script again.
+
+```shell
+# location of where to place the ImageNet data
+DATA_DIR=$HOME/imagenet-data
+
+# build the preprocessing script.
+bazel build slim/download_and_preprocess_imagenet
+
+# run it
+bazel-bin/slim/download_and_preprocess_imagenet "${DATA_DIR}"
+```
+
+The final line of the output script should read:
+
+```shell
+2016-02-17 14:30:17.287989: Finished writing all 1281167 images in data set.
+```
+
+When the script finishes you will find 1024 and 128 training and validation
+files in the `DATA_DIR`. The files will match the patterns `train-????-of-1024`
+and `validation-?????-of-00128`, respectively.
+
+[Congratulations!](https://www.youtube.com/watch?v=9bZkp7q19f0) You are now
+ready to train or evaluate with the ImageNet data set.
 
 # Pre-trained Models
 <a id='Pretrained'></a>

slim/__init__.py

-

slim/datasets/download_and_convert_imagenet.sh

+#!/bin/bash
+# Copyright 2016 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+# Script to download and preprocess ImageNet Challenge 2012
+# training and validation data set.
+#
+# The final output of this script are sharded TFRecord files containing
+# serialized Example protocol buffers. See build_imagenet_data.py for
+# details of how the Example protocol buffers contain the ImageNet data.
+#
+# The final output of this script appears as such:
+#
+#   data_dir/train-00000-of-01024
+#   data_dir/train-00001-of-01024
+#    ...
+#   data_dir/train-00127-of-01024
+#
+# and
+#
+#   data_dir/validation-00000-of-00128
+#   data_dir/validation-00001-of-00128
+#   ...
+#   data_dir/validation-00127-of-00128
+#
+# Note that this script may take several hours to run to completion. The
+# conversion of the ImageNet data to TFRecords alone takes 2-3 hours depending
+# on the speed of your machine. Please be patient.
+#
+# **IMPORTANT**
+# To download the raw images, the user must create an account with image-net.org
+# and generate a username and access_key. The latter two are required for
+# downloading the raw images.
+#
+# usage:
+#  ./download_and_preprocess_imagenet.sh [data-dir]
+set -e
+
+if [ -z "$1" ]; then
+  echo "usage download_and_preprocess_imagenet.sh [data dir]"
+  exit
+fi
+
+# Create the output and temporary directories.
+DATA_DIR="${1%/}"
+SCRATCH_DIR="${DATA_DIR}/raw-data/"
+mkdir -p "${DATA_DIR}"
+mkdir -p "${SCRATCH_DIR}"
+WORK_DIR="$0.runfiles/third_party/tensorflow_models/slim"
+
+# Download the ImageNet data.
+LABELS_FILE="${WORK_DIR}/datasets/imagenet_lsvrc_2015_synsets.txt"
+DOWNLOAD_SCRIPT="${WORK_DIR}/datasets/download_imagenet.sh"
+"${DOWNLOAD_SCRIPT}" "${SCRATCH_DIR}" "${LABELS_FILE}"
+
+# Note the locations of the train and validation data.
+TRAIN_DIRECTORY="${SCRATCH_DIR}train/"
+VALIDATION_DIRECTORY="${SCRATCH_DIR}validation/"
+
+# Preprocess the validation data by moving the images into the appropriate
+# sub-directory based on the label (synset) of the image.
+echo "Organizing the validation data into sub-directories."
+PREPROCESS_VAL_SCRIPT="${WORK_DIR}/datasets/preprocess_imagenet_validation_data.py"
+VAL_LABELS_FILE="${WORK_DIR}/datasets/imagenet_2012_validation_synset_labels.txt"
+
+"${PREPROCESS_VAL_SCRIPT}" "${VALIDATION_DIRECTORY}" "${VAL_LABELS_FILE}"
+
+# Convert the XML files for bounding box annotations into a single CSV.
+echo "Extracting bounding box information from XML."
+BOUNDING_BOX_SCRIPT="${WORK_DIR}/datasets/process_bounding_boxes.py"
+BOUNDING_BOX_FILE="${SCRATCH_DIR}/imagenet_2012_bounding_boxes.csv"
+BOUNDING_BOX_DIR="${SCRATCH_DIR}bounding_boxes/"
+
+"${BOUNDING_BOX_SCRIPT}" "${BOUNDING_BOX_DIR}" "${LABELS_FILE}" \
+ | sort >"${BOUNDING_BOX_FILE}"
+echo "Finished downloading and preprocessing the ImageNet data."
+
+# Build the TFRecords version of the ImageNet data.
+BUILD_SCRIPT="${WORK_DIR}/build_imagenet_data"
+OUTPUT_DIRECTORY="${DATA_DIR}"
+IMAGENET_METADATA_FILE="${WORK_DIR}/datasets/imagenet_metadata.txt"
+
+"${BUILD_SCRIPT}" \
+  --train_directory="${TRAIN_DIRECTORY}" \
+  --validation_directory="${VALIDATION_DIRECTORY}" \
+  --output_directory="${OUTPUT_DIRECTORY}" \
+  --imagenet_metadata_file="${IMAGENET_METADATA_FILE}" \
+  --labels_file="${LABELS_FILE}" \
+  --bounding_box_file="${BOUNDING_BOX_FILE}"

slim/datasets/download_imagenet.sh

+#!/bin/bash
+# Copyright 2016 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+# Script to download ImageNet Challenge 2012 training and validation data set.
+#
+# Downloads and decompresses raw images and bounding boxes.
+#
+# **IMPORTANT**
+# To download the raw images, the user must create an account with image-net.org
+# and generate a username and access_key. The latter two are required for
+# downloading the raw images.
+#
+# usage:
+#  ./download_imagenet.sh [dirname]
+set -e
+
+if [ "x$IMAGENET_ACCESS_KEY" == x -o "x$IMAGENET_USERNAME" == x ]; then
+  cat <<END
+In order to download the imagenet data, you have to create an account with
+image-net.org. This will get you a username and an access key. You can set the
+IMAGENET_USERNAME and IMAGENET_ACCESS_KEY environment variables, or you can
+enter the credentials here.
+END
+  read -p "Username: " IMAGENET_USERNAME
+  read -p "Access key: " IMAGENET_ACCESS_KEY
+fi
+
+OUTDIR="${1:-./imagenet-data}"
+SYNSETS_FILE="${2:-./synsets.txt}"
+
+echo "Saving downloaded files to $OUTDIR"
+mkdir -p "${OUTDIR}"
+CURRENT_DIR=$(pwd)
+BBOX_DIR="${OUTDIR}bounding_boxes"
+mkdir -p "${BBOX_DIR}"
+cd "${OUTDIR}"
+
+# Download and process all of the ImageNet bounding boxes.
+BASE_URL="http://www.image-net.org/challenges/LSVRC/2012/nonpub"
+
+# See here for details: http://www.image-net.org/download-bboxes
+BOUNDING_BOX_ANNOTATIONS="${BASE_URL}/ILSVRC2012_bbox_train_v2.tar.gz"
+BBOX_TAR_BALL="${BBOX_DIR}/annotations.tar.gz"
+echo "Downloading bounding box annotations."
+wget "${BOUNDING_BOX_ANNOTATIONS}" -O "${BBOX_TAR_BALL}"
+echo "Uncompressing bounding box annotations ..."
+tar xzf "${BBOX_TAR_BALL}" -C "${BBOX_DIR}"
+
+LABELS_ANNOTATED="${BBOX_DIR}/*"
+NUM_XML=$(ls -1 ${LABELS_ANNOTATED} | wc -l)
+echo "Identified ${NUM_XML} bounding box annotations."
+
+# Download and uncompress all images from the ImageNet 2012 validation dataset.
+VALIDATION_TARBALL="ILSVRC2012_img_val.tar"
+OUTPUT_PATH="${OUTDIR}validation/"
+mkdir -p "${OUTPUT_PATH}"
+cd "${OUTDIR}/.."
+echo "Downloading ${VALIDATION_TARBALL} to ${OUTPUT_PATH}."
+wget -nd -c "${BASE_URL}/${VALIDATION_TARBALL}"
+tar xf "${VALIDATION_TARBALL}" -C "${OUTPUT_PATH}"
+
+# Download all images from the ImageNet 2012 train dataset.
+TRAIN_TARBALL="ILSVRC2012_img_train.tar"
+OUTPUT_PATH="${OUTDIR}train/"
+mkdir -p "${OUTPUT_PATH}"
+cd "${OUTDIR}/.."
+echo "Downloading ${TRAIN_TARBALL} to ${OUTPUT_PATH}."
+wget -nd -c "${BASE_URL}/${TRAIN_TARBALL}"
+
+# Un-compress the individual tar-files within the train tar-file.
+echo "Uncompressing individual train tar-balls in the training data."
+
+while read SYNSET; do
+  echo "Processing: ${SYNSET}"
+
+  # Create a directory and delete anything there.
+  mkdir -p "${OUTPUT_PATH}/${SYNSET}"
+  rm -rf "${OUTPUT_PATH}/${SYNSET}/*"
+
+  # Uncompress into the directory.
+  tar xf "${TRAIN_TARBALL}" "${SYNSET}.tar"
+  tar xf "${SYNSET}.tar" -C "${OUTPUT_PATH}/${SYNSET}/"
+  rm -f "${SYNSET}.tar"
+
+  echo "Finished processing: ${SYNSET}"
+done < "${SYNSETS_FILE}"

slim/datasets/imagenet_lsvrc_2015_synsets.txt

+n01440764
+n01443537
+n01484850
+n01491361
+n01494475
+n01496331
+n01498041
+n01514668
+n01514859
+n01518878
+n01530575
+n01531178
+n01532829
+n01534433
+n01537544
+n01558993
+n01560419
+n01580077
+n01582220
+n01592084
+n01601694
+n01608432
+n01614925
+n01616318
+n01622779
+n01629819
+n01630670
+n01631663
+n01632458
+n01632777
+n01641577
+n01644373
+n01644900
+n01664065
+n01665541
+n01667114
+n01667778
+n01669191
+n01675722
+n01677366
+n01682714
+n01685808
+n01687978
+n01688243
+n01689811
+n01692333
+n01693334
+n01694178
+n01695060
+n01697457
+n01698640
+n01704323
+n01728572
+n01728920
+n01729322
+n01729977
+n01734418
+n01735189
+n01737021
+n01739381
+n01740131
+n01742172
+n01744401
+n01748264
+n01749939
+n01751748
+n01753488
+n01755581
+n01756291
+n01768244
+n01770081
+n01770393
+n01773157
+n01773549
+n01773797
+n01774384
+n01774750
+n01775062
+n01776313
+n01784675
+n01795545
+n01796340
+n01797886
+n01798484
+n01806143
+n01806567
+n01807496
+n01817953
+n01818515
+n01819313
+n01820546
+n01824575
+n01828970
+n01829413
+n01833805
+n01843065
+n01843383
+n01847000
+n01855032
+n01855672
+n01860187
+n01871265
+n01872401
+n01873310
+n01877812
+n01882714
+n01883070
+n01910747
+n01914609
+n01917289
+n01924916
+n01930112
+n01943899
+n01944390
+n01945685
+n01950731
+n01955084
+n01968897
+n01978287
+n01978455
+n01980166
+n01981276
+n01983481
+n01984695
+n01985128
+n01986214
+n01990800
+n02002556
+n02002724
+n02006656
+n02007558
+n02009229
+n02009912
+n02011460
+n02012849
+n02013706
+n02017213
+n02018207
+n02018795
+n02025239
+n02027492
+n02028035
+n02033041
+n02037110
+n02051845
+n02056570
+n02058221
+n02066245
+n02071294
+n02074367
+n02077923
+n02085620
+n02085782
+n02085936
+n02086079
+n02086240
+n02086646
+n02086910
+n02087046
+n02087394
+n02088094
+n02088238
+n02088364
+n02088466
+n02088632
+n02089078
+n02089867
+n02089973
+n02090379
+n02090622
+n02090721
+n02091032
+n02091134
+n02091244
+n02091467
+n02091635
+n02091831
+n02092002
+n02092339
+n02093256
+n02093428
+n02093647
+n02093754
+n02093859
+n02093991
+n02094114
+n02094258
+n02094433
+n02095314
+n02095570
+n02095889
+n02096051
+n02096177
+n02096294
+n02096437
+n02096585
+n02097047
+n02097130
+n02097209
+n02097298
+n02097474
+n02097658
+n02098105
+n02098286
+n02098413
+n02099267
+n02099429
+n02099601
+n02099712
+n02099849
+n02100236
+n02100583
+n02100735
+n02100877
+n02101006
+n02101388
+n02101556
+n02102040
+n02102177
+n02102318
+n02102480
+n02102973
+n02104029
+n02104365
+n02105056
+n02105162
+n02105251
+n02105412
+n02105505
+n02105641
+n02105855
+n02106030
+n02106166
+n02106382
+n02106550
+n02106662
+n02107142
+n02107312
+n02107574
+n02107683
+n02107908
+n02108000
+n02108089
+n02108422
+n02108551
+n02108915
+n02109047
+n02109525
+n02109961
+n02110063
+n02110185
+n02110341
+n02110627
+n02110806
+n02110958
+n02111129
+n02111277
+n02111500
+n02111889
+n02112018
+n02112137
+n02112350
+n02112706
+n02113023
+n02113186
+n02113624
+n02113712
+n02113799
+n02113978
+n02114367
+n02114548
+n02114712
+n02114855
+n02115641
+n02115913
+n02116738
+n02117135
+n02119022
+n02119789
+n02120079
+n02120505
+n02123045
+n02123159
+n02123394
+n02123597
+n02124075
+n02125311
+n02127052
+n02128385
+n02128757
+n02128925
+n02129165
+n02129604
+n02130308
+n02132136
+n02133161
+n02134084
+n02134418
+n02137549
+n02138441
+n02165105
+n02165456
+n02167151
+n02168699
+n02169497
+n02172182
+n02174001
+n02177972
+n02190166
+n02206856
+n02219486
+n02226429
+n02229544
+n02231487
+n02233338
+n02236044
+n02256656
+n02259212
+n02264363
+n02268443
+n02268853
+n02276258
+n02277742
+n02279972
+n02280649
+n02281406
+n02281787
+n02317335
+n02319095
+n02321529
+n02325366
+n02326432
+n02328150
+n02342885
+n02346627
+n02356798
+n02361337
+n02363005
+n02364673
+n02389026
+n02391049
+n02395406
+n02396427
+n02397096
+n02398521
+n02403003
+n02408429
+n02410509
+n02412080
+n02415577
+n02417914
+n02422106
+n02422699
+n02423022
+n02437312
+n02437616
+n02441942
+n02442845
+n02443114
+n02443484
+n02444819
+n02445715
+n02447366
+n02454379
+n02457408
+n02480495
+n02480855
+n02481823
+n02483362
+n02483708
+n02484975
+n02486261
+n02486410
+n02487347
+n02488291
+n02488702
+n02489166
+n02490219
+n02492035
+n02492660
+n02493509
+n02493793
+n02494079
+n02497673
+n02500267
+n02504013
+n02504458
+n02509815
+n02510455
+n02514041
+n02526121
+n02536864
+n02606052
+n02607072
+n02640242
+n02641379
+n02643566
+n02655020
+n02666196
+n02667093
+n02669723
+n02672831
+n02676566
+n02687172
+n02690373
+n02692877
+n02699494
+n02701002
+n02704792
+n02708093
+n02727426
+n02730930
+n02747177
+n02749479
+n02769748
+n02776631
+n02777292
+n02782093
+n02783161
+n02786058
+n02787622
+n02788148
+n02790996
+n02791124
+n02791270
+n02793495
+n02794156
+n02795169
+n02797295
+n02799071
+n02802426
+n02804414
+n02804610
+n02807133
+n02808304
+n02808440
+n02814533
+n02814860
+n02815834
+n02817516
+n02823428
+n02823750
+n02825657
+n02834397
+n02835271
+n02837789
+n02840245
+n02841315
+n02843684
+n02859443
+n02860847
+n02865351
+n02869837
+n02870880
+n02871525
+n02877765
+n02879718
+n02883205
+n02892201
+n02892767
+n02894605
+n02895154
+n02906734
+n02909870
+n02910353
+n02916936
+n02917067
+n02927161
+n02930766
+n02939185
+n02948072
+n02950826
+n02951358
+n02951585
+n02963159
+n02965783
+n02966193
+n02966687
+n02971356
+n02974003
+n02977058
+n02978881
+n02979186
+n02980441
+n02981792
+n02988304
+n02992211
+n02992529
+n02999410
+n03000134
+n03000247
+n03000684
+n03014705
+n03016953
+n03017168
+n03018349
+n03026506
+n03028079
+n03032252
+n03041632
+n03042490
+n03045698
+n03047690
+n03062245
+n03063599
+n03063689
+n03065424
+n03075370
+n03085013
+n03089624
+n03095699
+n03100240
+n03109150
+n03110669
+n03124043
+n03124170
+n03125729
+n03126707
+n03127747
+n03127925
+n03131574
+n03133878
+n03134739
+n03141823
+n03146219
+n03160309
+n03179701
+n03180011
+n03187595
+n03188531
+n03196217
+n03197337
+n03201208
+n03207743
+n03207941
+n03208938
+n03216828
+n03218198
+n03220513
+n03223299
+n03240683
+n03249569
+n03250847
+n03255030
+n03259280
+n03271574
+n03272010
+n03272562
+n03290653
+n03291819
+n03297495
+n03314780
+n03325584
+n03337140
+n03344393
+n03345487
+n03347037
+n03355925
+n03372029
+n03376595
+n03379051
+n03384352
+n03388043
+n03388183
+n03388549
+n03393912
+n03394916
+n03400231
+n03404251
+n03417042
+n03424325
+n03425413
+n03443371
+n03444034
+n03445777
+n03445924
+n03447447
+n03447721
+n03450230
+n03452741
+n03457902
+n03459775
+n03461385
+n03467068
+n03476684
+n03476991
+n03478589
+n03481172
+n03482405
+n03483316
+n03485407
+n03485794
+n03492542
+n03494278
+n03495258
+n03496892
+n03498962
+n03527444
+n03529860
+n03530642
+n03532672
+n03534580
+n03535780
+n03538406
+n03544143
+n03584254
+n03584829
+n03590841
+n03594734
+n03594945
+n03595614
+n03598930
+n03599486
+n03602883
+n03617480
+n03623198
+n03627232
+n03630383
+n03633091
+n03637318
+n03642806
+n03649909
+n03657121
+n03658185
+n03661043
+n03662601
+n03666591
+n03670208
+n03673027
+n03676483
+n03680355
+n03690938
+n03691459
+n03692522
+n03697007
+n03706229
+n03709823
+n03710193
+n03710637
+n03710721
+n03717622
+n03720891
+n03721384
+n03724870
+n03729826
+n03733131
+n03733281
+n03733805
+n03742115
+n03743016
+n03759954
+n03761084
+n03763968
+n03764736
+n03769881
+n03770439
+n03770679
+n03773504
+n03775071
+n03775546
+n03776460
+n03777568
+n03777754
+n03781244
+n03782006
+n03785016
+n03786901
+n03787032
+n03788195
+n03788365
+n03791053
+n03792782
+n03792972
+n03793489
+n03794056
+n03796401
+n03803284
+n03804744
+n03814639
+n03814906
+n03825788
+n03832673
+n03837869
+n03838899
+n03840681
+n03841143
+n03843555
+n03854065
+n03857828
+n03866082
+n03868242
+n03868863
+n03871628
+n03873416
+n03874293
+n03874599
+n03876231
+n03877472
+n03877845
+n03884397
+n03887697
+n03888257
+n03888605
+n03891251
+n03891332
+n03895866
+n03899768
+n03902125
+n03903868
+n03908618
+n03908714
+n03916031
+n03920288
+n03924679
+n03929660
+n03929855
+n03930313
+n03930630
+n03933933
+n03935335
+n03937543
+n03938244
+n03942813
+n03944341
+n03947888
+n03950228
+n03954731
+n03956157
+n03958227
+n03961711
+n03967562
+n03970156
+n03976467
+n03976657
+n03977966
+n03980874
+n03982430
+n03983396
+n03991062
+n03992509
+n03995372
+n03998194
+n04004767
+n04005630
+n04008634
+n04009552
+n04019541
+n04023962
+n04026417
+n04033901
+n04033995
+n04037443
+n04039381
+n04040759
+n04041544
+n04044716
+n04049303
+n04065272
+n04067472
+n04069434
+n04070727
+n04074963
+n04081281
+n04086273
+n04090263
+n04099969
+n04111531
+n04116512
+n04118538
+n04118776
+n04120489
+n04125021
+n04127249
+n04131690
+n04133789
+n04136333
+n04141076
+n04141327
+n04141975
+n04146614
+n04147183
+n04149813
+n04152593
+n04153751
+n04154565
+n04162706
+n04179913
+n04192698
+n04200800
+n04201297
+n04204238
+n04204347
+n04208210
+n04209133
+n04209239
+n04228054
+n04229816
+n04235860
+n04238763
+n04239074
+n04243546
+n04251144
+n04252077
+n04252225
+n04254120
+n04254680
+n04254777
+n04258138
+n04259630
+n04263257
+n04264628
+n04265275
+n04266014
+n04270147
+n04273569
+n04275548
+n04277352
+n04285008
+n04286575
+n04296562
+n04310018
+n04311004
+n04311174
+n04317175
+n04325704
+n04326547
+n04328186
+n04330267
+n04332243
+n04335435
+n04336792
+n04344873
+n04346328
+n04347754
+n04350905
+n04355338
+n04355933
+n04356056
+n04357314
+n04366367
+n04367480
+n04370456
+n04371430
+n04371774
+n04372370
+n04376876
+n04380533
+n04389033
+n04392985
+n04398044
+n04399382
+n04404412
+n04409515
+n04417672
+n04418357
+n04423845
+n04428191
+n04429376
+n04435653
+n04442312
+n04443257
+n04447861
+n04456115
+n04458633
+n04461696
+n04462240
+n04465501
+n04467665
+n04476259
+n04479046
+n04482393
+n04483307
+n04485082
+n04486054
+n04487081
+n04487394
+n04493381
+n04501370
+n04505470
+n04507155
+n04509417
+n04515003
+n04517823
+n04522168
+n04523525
+n04525038
+n04525305
+n04532106
+n04532670
+n04536866
+n04540053
+n04542943
+n04548280
+n04548362
+n04550184
+n04552348
+n04553703
+n04554684
+n04557648
+n04560804
+n04562935
+n04579145
+n04579432
+n04584207
+n04589890
+n04590129
+n04591157
+n04591713
+n04592741
+n04596742
+n04597913
+n04599235
+n04604644
+n04606251
+n04612504
+n04613696
+n06359193
+n06596364
+n06785654
+n06794110
+n06874185
+n07248320
+n07565083
+n07579787
+n07583066
+n07584110
+n07590611
+n07613480
+n07614500
+n07615774
+n07684084
+n07693725
+n07695742
+n07697313
+n07697537
+n07711569
+n07714571
+n07714990
+n07715103
+n07716358
+n07716906
+n07717410
+n07717556
+n07718472
+n07718747
+n07720875
+n07730033
+n07734744
+n07742313
+n07745940
+n07747607
+n07749582
+n07753113
+n07753275
+n07753592
+n07754684
+n07760859
+n07768694
+n07802026
+n07831146
+n07836838
+n07860988
+n07871810
+n07873807
+n07875152
+n07880968
+n07892512
+n07920052
+n07930864
+n07932039
+n09193705
+n09229709
+n09246464
+n09256479
+n09288635
+n09332890
+n09399592
+n09421951
+n09428293
+n09468604
+n09472597
+n09835506
+n10148035
+n10565667
+n11879895
+n11939491
+n12057211
+n12144580
+n12267677
+n12620546
+n12768682
+n12985857
+n12998815
+n13037406
+n13040303
+n13044778
+n13052670
+n13054560
+n13133613
+n15075141

slim/datasets/preprocess_imagenet_validation_data.py

+#!/usr/bin/python
+# Copyright 2016 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Process the ImageNet Challenge bounding boxes for TensorFlow model training.
+
+Associate the ImageNet 2012 Challenge validation data set with labels.
+
+The raw ImageNet validation data set is expected to reside in JPEG files
+located in the following directory structure.
+
+ data_dir/ILSVRC2012_val_00000001.JPEG
+ data_dir/ILSVRC2012_val_00000002.JPEG
+ ...
+ data_dir/ILSVRC2012_val_00050000.JPEG
+
+This script moves the files into a directory structure like such:
+ data_dir/n01440764/ILSVRC2012_val_00000293.JPEG
+ data_dir/n01440764/ILSVRC2012_val_00000543.JPEG
+ ...
+where 'n01440764' is the unique synset label associated with
+these images.
+
+This directory reorganization requires a mapping from validation image
+number (i.e. suffix of the original file) to the associated label. This
+is provided in the ImageNet development kit via a Matlab file.
+
+In order to make life easier and divorce ourselves from Matlab, we instead
+supply a custom text file that provides this mapping for us.
+
+Sample usage:
+  ./preprocess_imagenet_validation_data.py ILSVRC2012_img_val \
+  imagenet_2012_validation_synset_labels.txt
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+import os.path
+import sys
+
+
+if __name__ == '__main__':
+  if len(sys.argv) < 3:
+    print('Invalid usage\n'
+          'usage: preprocess_imagenet_validation_data.py '
+          '<validation data dir> <validation labels file>')
+    sys.exit(-1)
+  data_dir = sys.argv[1]
+  validation_labels_file = sys.argv[2]
+
+  # Read in the 50000 synsets associated with the validation data set.
+  labels = [l.strip() for l in open(validation_labels_file).readlines()]
+  unique_labels = set(labels)
+
+  # Make all sub-directories in the validation data dir.
+  for label in unique_labels:
+    labeled_data_dir = os.path.join(data_dir, label)
+    os.makedirs(labeled_data_dir)
+
+  # Move all of the image to the appropriate sub-directory.
+  for i in xrange(len(labels)):
+    basename = 'ILSVRC2012_val_000%.5d.JPEG' % (i + 1)
+    original_filename = os.path.join(data_dir, basename)
+    if not os.path.exists(original_filename):
+      print('Failed to find: ' % original_filename)
+      sys.exit(-1)
+    new_filename = os.path.join(data_dir, labels[i], basename)
+    os.rename(original_filename, new_filename)

slim/datasets/process_bounding_boxes.py

+#!/usr/bin/python
+# Copyright 2016 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Process the ImageNet Challenge bounding boxes for TensorFlow model training.
+
+This script is called as
+
+process_bounding_boxes.py <dir> [synsets-file]
+
+Where <dir> is a directory containing the downloaded and unpacked bounding box
+data. If [synsets-file] is supplied, then only the bounding boxes whose
+synstes are contained within this file are returned. Note that the
+[synsets-file] file contains synset ids, one per line.
+
+The script dumps out a CSV text file in which each line contains an entry.
+  n00007846_64193.JPEG,0.0060,0.2620,0.7545,0.9940
+
+The entry can be read as:
+  <JPEG file name>, <xmin>, <ymin>, <xmax>, <ymax>
+
+The bounding box for <JPEG file name> contains two points (xmin, ymin) and
+(xmax, ymax) specifying the lower-left corner and upper-right corner of a
+bounding box in *relative* coordinates.
+
+The user supplies a directory where the XML files reside. The directory
+structure in the directory <dir> is assumed to look like this:
+
+<dir>/nXXXXXXXX/nXXXXXXXX_YYYY.xml
+
+Each XML file contains a bounding box annotation. The script:
+
+ (1) Parses the XML file and extracts the filename, label and bounding box info.
+
+ (2) The bounding box is specified in the XML files as integer (xmin, ymin) and
+    (xmax, ymax) *relative* to image size displayed to the human annotator. The
+    size of the image displayed to the human annotator is stored in the XML file
+    as integer (height, width).
+
+    Note that the displayed size will differ from the actual size of the image
+    downloaded from image-net.org. To make the bounding box annotation useable,
+    we convert bounding box to floating point numbers relative to displayed
+    height and width of the image.
+
+    Note that each XML file might contain N bounding box annotations.
+
+    Note that the points are all clamped at a range of [0.0, 1.0] because some
+    human annotations extend outside the range of the supplied image.
+
+    See details here: http://image-net.org/download-bboxes
+
+(3) By default, the script outputs all valid bounding boxes. If a
+    [synsets-file] is supplied, only the subset of bounding boxes associated
+    with those synsets are outputted. Importantly, one can supply a list of
+    synsets in the ImageNet Challenge and output the list of bounding boxes
+    associated with the training images of the ILSVRC.
+
+    We use these bounding boxes to inform the random distortion of images
+    supplied to the network.
+
+If you run this script successfully, you will see the following output
+to stderr:
+> Finished processing 544546 XML files.
+> Skipped 0 XML files not in ImageNet Challenge.
+> Skipped 0 bounding boxes not in ImageNet Challenge.
+> Wrote 615299 bounding boxes from 544546 annotated images.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import glob
+import os.path
+import sys
+import xml.etree.ElementTree as ET
+
+
+class BoundingBox(object):
+  pass
+
+
+def GetItem(name, root, index=0):
+  count = 0
+  for item in root.iter(name):
+    if count == index:
+      return item.text
+    count += 1
+  # Failed to find "index" occurrence of item.
+  return -1
+
+
+def GetInt(name, root, index=0):
+  return int(GetItem(name, root, index))
+
+
+def FindNumberBoundingBoxes(root):
+  index = 0
+  while True:
+    if GetInt('xmin', root, index) == -1:
+      break
+    index += 1
+  return index
+
+
+def ProcessXMLAnnotation(xml_file):
+  """Process a single XML file containing a bounding box."""
+  # pylint: disable=broad-except
+  try:
+    tree = ET.parse(xml_file)
+  except Exception:
+    print('Failed to parse: ' + xml_file, file=sys.stderr)
+    return None
+  # pylint: enable=broad-except
+  root = tree.getroot()
+
+  num_boxes = FindNumberBoundingBoxes(root)
+  boxes = []
+
+  for index in xrange(num_boxes):
+    box = BoundingBox()
+    # Grab the 'index' annotation.
+    box.xmin = GetInt('xmin', root, index)
+    box.ymin = GetInt('ymin', root, index)
+    box.xmax = GetInt('xmax', root, index)
+    box.ymax = GetInt('ymax', root, index)
+
+    box.width = GetInt('width', root)
+    box.height = GetInt('height', root)
+    box.filename = GetItem('filename', root) + '.JPEG'
+    box.label = GetItem('name', root)
+
+    xmin = float(box.xmin) / float(box.width)
+    xmax = float(box.xmax) / float(box.width)
+    ymin = float(box.ymin) / float(box.height)
+    ymax = float(box.ymax) / float(box.height)
+
+    # Some images contain bounding box annotations that
+    # extend outside of the supplied image. See, e.g.
+    # n03127925/n03127925_147.xml
+    # Additionally, for some bounding boxes, the min > max
+    # or the box is entirely outside of the image.
+    min_x = min(xmin, xmax)
+    max_x = max(xmin, xmax)
+    box.xmin_scaled = min(max(min_x, 0.0), 1.0)
+    box.xmax_scaled = min(max(max_x, 0.0), 1.0)
+
+    min_y = min(ymin, ymax)
+    max_y = max(ymin, ymax)
+    box.ymin_scaled = min(max(min_y, 0.0), 1.0)
+    box.ymax_scaled = min(max(max_y, 0.0), 1.0)
+
+    boxes.append(box)
+
+  return boxes
+
+if __name__ == '__main__':
+  if len(sys.argv) < 2 or len(sys.argv) > 3:
+    print('Invalid usage\n'
+          'usage: process_bounding_boxes.py <dir> [synsets-file]',
+          file=sys.stderr)
+    sys.exit(-1)
+
+  xml_files = glob.glob(sys.argv[1] + '/*/*.xml')
+  print('Identified %d XML files in %s' % (len(xml_files), sys.argv[1]),
+        file=sys.stderr)
+
+  if len(sys.argv) == 3:
+    labels = set([l.strip() for l in open(sys.argv[2]).readlines()])
+    print('Identified %d synset IDs in %s' % (len(labels), sys.argv[2]),
+          file=sys.stderr)
+  else:
+    labels = None
+
+  skipped_boxes = 0
+  skipped_files = 0
+  saved_boxes = 0
+  saved_files = 0
+  for file_index, one_file in enumerate(xml_files):
+    # Example: <...>/n06470073/n00141669_6790.xml
+    label = os.path.basename(os.path.dirname(one_file))
+
+    # Determine if the annotation is from an ImageNet Challenge label.
+    if labels is not None and label not in labels:
+      skipped_files += 1
+      continue
+
+    bboxes = ProcessXMLAnnotation(one_file)
+    assert bboxes is not None, 'No bounding boxes found in ' + one_file
+
+    found_box = False
+    for bbox in bboxes:
+      if labels is not None:
+        if bbox.label != label:
+          # Note: There is a slight bug in the bounding box annotation data.
+          # Many of the dog labels have the human label 'Scottish_deerhound'
+          # instead of the synset ID 'n02092002' in the bbox.label field. As a
+          # simple hack to overcome this issue, we only exclude bbox labels
+          # *which are synset ID's* that do not match original synset label for
+          # the XML file.
+          if bbox.label in labels:
+            skipped_boxes += 1
+            continue
+
+      # Guard against improperly specified boxes.
+      if (bbox.xmin_scaled >= bbox.xmax_scaled or
+          bbox.ymin_scaled >= bbox.ymax_scaled):
+        skipped_boxes += 1
+        continue
+
+      # Note bbox.filename occasionally contains '%s' in the name. This is
+      # data set noise that is fixed by just using the basename of the XML file.
+      image_filename = os.path.splitext(os.path.basename(one_file))[0]
+      print('%s.JPEG,%.4f,%.4f,%.4f,%.4f' %
+            (image_filename,
+             bbox.xmin_scaled, bbox.ymin_scaled,
+             bbox.xmax_scaled, bbox.ymax_scaled))
+
+      saved_boxes += 1
+      found_box = True
+    if found_box:
+      saved_files += 1
+    else:
+      skipped_files += 1
+
+    if not file_index % 5000:
+      print('--> processed %d of %d XML files.' %
+            (file_index + 1, len(xml_files)),
+            file=sys.stderr)
+      print('--> skipped %d boxes and %d XML files.' %
+            (skipped_boxes, skipped_files), file=sys.stderr)
+
+  print('Finished processing %d XML files.' % len(xml_files), file=sys.stderr)
+  print('Skipped %d XML files not in ImageNet Challenge.' % skipped_files,
+        file=sys.stderr)
+  print('Skipped %d bounding boxes not in ImageNet Challenge.' % skipped_boxes,
+        file=sys.stderr)
+  print('Wrote %d bounding boxes from %d annotated images.' %
+        (saved_boxes, saved_files),
+        file=sys.stderr)
+  print('Finished.', file=sys.stderr)