Skip to content

GitLab

Commit ba87e2c6 authored by Mark Sandler's avatar Mark Sandler Committed by Sergio Guadarrama
Browse files

Merged commit includes the following changes: (#7797) 

279978375  by Sergio Guadarrama:

    Pass s=2 to the expanded_conv block so it can apply residual correctly in case of fused convolutions.  (Before it was relying on channel mismatch only)

--
279788358  by Sergio Guadarrama:

    Update README to add mobilenet-edgetpu details

--
279774392  by Sergio Guadarrama:

    Adds MobilenetV3-EdgeTpu definition.

--
278917344  by Sergio Guadarrama:

    Create visualwakewords dataset using slim scripts instead of custom scripts.

--
277940048  by Sergio Guadarrama:

    Internal changes to tf.contrib symbols

--

PiperOrigin-RevId: 279978375
parent c2902cfb
Show whitespace changes
Inline Side-by-side
Showing with 676 additions and 306 deletions
research/slim/BUILD
View file @ ba87e2c6
...@@ -39,6 +39,7 @@ py_binary( ...@@ -39,6 +39,7 @@ py_binary(
python_version = "PY2", python_version = "PY2",
deps = [ deps = [
# "//numpy", # "//numpy",
"//third_party/py/six",
# "//tensorflow", # "//tensorflow",
], ],
) )
...@@ -49,6 +50,7 @@ py_library( ...@@ -49,6 +50,7 @@ py_library(
deps = [ deps = [
":dataset_utils", ":dataset_utils",
# "//numpy", # "//numpy",
"//third_party/py/six",
# "//tensorflow", # "//tensorflow",
], ],
) )
...@@ -68,44 +70,40 @@ py_library( ...@@ -68,44 +70,40 @@ py_library(
deps = [ deps = [
":dataset_utils", ":dataset_utils",
# "//numpy", # "//numpy",
"//third_party/py/six",
# "//tensorflow", # "//tensorflow",
], ],
) )
py_binary( py_library(
name = "download_and_convert_data", name = "download_and_convert_visualwakewords_lib",
srcs = ["download_and_convert_data.py"], srcs = ["datasets/download_and_convert_visualwakewords_lib.py"],
python_version = "PY2",
deps = [ deps = [
":download_and_convert_cifar10", ":dataset_utils",
":download_and_convert_flowers", "//third_party/py/PIL:pil",
":download_and_convert_mnist", "//third_party/py/contextlib2",
# "//tensorflow", # "//tensorflow",
], ],
) )
sh_binary( py_library(
name = "download_mscoco", name = "download_and_convert_visualwakewords",
srcs = ["datasets/download_mscoco.sh"], srcs = ["datasets/download_and_convert_visualwakewords.py"],
)
py_binary(
name = "build_visualwakewords_data",
srcs = ["datasets/build_visualwakewords_data.py"],
python_version = "PY2",
deps = [ deps = [
":build_visualwakewords_data_lib", ":download_and_convert_visualwakewords_lib",
# "//tensorflow", # "//tensorflow",
], ],
) )
py_library( py_binary(
name = "build_visualwakewords_data_lib", name = "download_and_convert_data",
srcs = ["datasets/build_visualwakewords_data_lib.py"], srcs = ["download_and_convert_data.py"],
python_version = "PY2",
deps = [ deps = [
":dataset_utils", ":download_and_convert_cifar10",
"//third_party/py/PIL:pil", ":download_and_convert_flowers",
"//third_party/py/contextlib2", ":download_and_convert_mnist",
":download_and_convert_visualwakewords",
# "//tensorflow", # "//tensorflow",
], ],
) )
...@@ -116,6 +114,7 @@ py_library( ...@@ -116,6 +114,7 @@ py_library(
deps = [ deps = [
":dataset_utils", ":dataset_utils",
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -125,6 +124,7 @@ py_library( ...@@ -125,6 +124,7 @@ py_library(
deps = [ deps = [
":dataset_utils", ":dataset_utils",
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -133,7 +133,9 @@ py_library( ...@@ -133,7 +133,9 @@ py_library(
srcs = ["datasets/imagenet.py"], srcs = ["datasets/imagenet.py"],
deps = [ deps = [
":dataset_utils", ":dataset_utils",
"//third_party/py/six",
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -143,6 +145,7 @@ py_library( ...@@ -143,6 +145,7 @@ py_library(
deps = [ deps = [
":dataset_utils", ":dataset_utils",
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -152,6 +155,7 @@ py_library( ...@@ -152,6 +155,7 @@ py_library(
deps = [ deps = [
":dataset_utils", ":dataset_utils",
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -172,6 +176,7 @@ py_library( ...@@ -172,6 +176,7 @@ py_library(
srcs = ["deployment/model_deploy.py"], srcs = ["deployment/model_deploy.py"],
deps = [ deps = [
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -184,6 +189,9 @@ py_test( ...@@ -184,6 +189,9 @@ py_test(
":model_deploy", ":model_deploy",
# "//numpy", # "//numpy",
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/framework:framework_py",
# "//tensorflow/contrib/layers:layers_py",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -192,6 +200,7 @@ py_library( ...@@ -192,6 +200,7 @@ py_library(
srcs = ["preprocessing/cifarnet_preprocessing.py"], srcs = ["preprocessing/cifarnet_preprocessing.py"],
deps = [ deps = [
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -209,6 +218,7 @@ py_library( ...@@ -209,6 +218,7 @@ py_library(
srcs = ["preprocessing/lenet_preprocessing.py"], srcs = ["preprocessing/lenet_preprocessing.py"],
deps = [ deps = [
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -217,6 +227,7 @@ py_library( ...@@ -217,6 +227,7 @@ py_library(
srcs = ["preprocessing/vgg_preprocessing.py"], srcs = ["preprocessing/vgg_preprocessing.py"],
deps = [ deps = [
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -228,7 +239,7 @@ py_library( ...@@ -228,7 +239,7 @@ py_library(
":inception_preprocessing", ":inception_preprocessing",
":lenet_preprocessing", ":lenet_preprocessing",
":vgg_preprocessing", ":vgg_preprocessing",
# "//tensorflow", # "//tensorflow/contrib/slim",
], ],
) )
...@@ -261,6 +272,7 @@ py_library( ...@@ -261,6 +272,7 @@ py_library(
srcs_version = "PY2AND3", srcs_version = "PY2AND3",
deps = [ deps = [
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -273,6 +285,7 @@ py_test( ...@@ -273,6 +285,7 @@ py_test(
deps = [ deps = [
":alexnet", ":alexnet",
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -281,6 +294,7 @@ py_library( ...@@ -281,6 +294,7 @@ py_library(
srcs = ["nets/cifarnet.py"], srcs = ["nets/cifarnet.py"],
deps = [ deps = [
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -289,7 +303,11 @@ py_library( ...@@ -289,7 +303,11 @@ py_library(
srcs = ["nets/cyclegan.py"], srcs = ["nets/cyclegan.py"],
deps = [ deps = [
# "//numpy", # "//numpy",
"//third_party/py/six",
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/framework:framework_py",
# "//tensorflow/contrib/layers:layers_py",
# "//tensorflow/contrib/util:util_py",
], ],
) )
...@@ -309,7 +327,9 @@ py_library( ...@@ -309,7 +327,9 @@ py_library(
name = "dcgan", name = "dcgan",
srcs = ["nets/dcgan.py"], srcs = ["nets/dcgan.py"],
deps = [ deps = [
"//third_party/py/six",
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -321,6 +341,7 @@ py_test( ...@@ -321,6 +341,7 @@ py_test(
srcs_version = "PY2AND3", srcs_version = "PY2AND3",
deps = [ deps = [
":dcgan", ":dcgan",
"//third_party/py/six",
# "//tensorflow", # "//tensorflow",
], ],
) )
...@@ -333,6 +354,7 @@ py_library( ...@@ -333,6 +354,7 @@ py_library(
":i3d_utils", ":i3d_utils",
":s3dg", ":s3dg",
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -354,7 +376,10 @@ py_library( ...@@ -354,7 +376,10 @@ py_library(
srcs = ["nets/i3d_utils.py"], srcs = ["nets/i3d_utils.py"],
srcs_version = "PY2AND3", srcs_version = "PY2AND3",
deps = [ deps = [
# "//numpy",
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/framework:framework_py",
# "//tensorflow/contrib/layers:layers_py",
], ],
) )
...@@ -377,6 +402,7 @@ py_library( ...@@ -377,6 +402,7 @@ py_library(
srcs_version = "PY2AND3", srcs_version = "PY2AND3",
deps = [ deps = [
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -387,6 +413,7 @@ py_library( ...@@ -387,6 +413,7 @@ py_library(
deps = [ deps = [
":inception_utils", ":inception_utils",
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -397,6 +424,7 @@ py_library( ...@@ -397,6 +424,7 @@ py_library(
deps = [ deps = [
":inception_utils", ":inception_utils",
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -407,6 +435,7 @@ py_library( ...@@ -407,6 +435,7 @@ py_library(
deps = [ deps = [
":inception_utils", ":inception_utils",
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -417,6 +446,7 @@ py_library( ...@@ -417,6 +446,7 @@ py_library(
deps = [ deps = [
":inception_utils", ":inception_utils",
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -426,6 +456,7 @@ py_library( ...@@ -426,6 +456,7 @@ py_library(
srcs_version = "PY2AND3", srcs_version = "PY2AND3",
deps = [ deps = [
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -440,6 +471,7 @@ py_test( ...@@ -440,6 +471,7 @@ py_test(
":inception", ":inception",
# "//numpy", # "//numpy",
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -454,6 +486,7 @@ py_test( ...@@ -454,6 +486,7 @@ py_test(
":inception", ":inception",
# "//numpy", # "//numpy",
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -468,6 +501,7 @@ py_test( ...@@ -468,6 +501,7 @@ py_test(
":inception", ":inception",
# "//numpy", # "//numpy",
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -481,6 +515,7 @@ py_test( ...@@ -481,6 +515,7 @@ py_test(
deps = [ deps = [
":inception", ":inception",
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -494,6 +529,7 @@ py_test( ...@@ -494,6 +529,7 @@ py_test(
deps = [ deps = [
":inception", ":inception",
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -502,6 +538,7 @@ py_library( ...@@ -502,6 +538,7 @@ py_library(
srcs = ["nets/lenet.py"], srcs = ["nets/lenet.py"],
deps = [ deps = [
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -511,6 +548,8 @@ py_library( ...@@ -511,6 +548,8 @@ py_library(
srcs_version = "PY2AND3", srcs_version = "PY2AND3",
deps = [ deps = [
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/layers:layers_py",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -523,6 +562,7 @@ py_library( ...@@ -523,6 +562,7 @@ py_library(
srcs_version = "PY2AND3", srcs_version = "PY2AND3",
deps = [ deps = [
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -533,6 +573,8 @@ py_library( ...@@ -533,6 +573,8 @@ py_library(
deps = [ deps = [
":mobilenet_common", ":mobilenet_common",
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/layers:layers_py",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -542,7 +584,9 @@ py_library( ...@@ -542,7 +584,9 @@ py_library(
srcs_version = "PY2AND3", srcs_version = "PY2AND3",
deps = [ deps = [
":mobilenet_common", ":mobilenet_common",
# "//numpy",
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -553,7 +597,9 @@ py_test( ...@@ -553,7 +597,9 @@ py_test(
srcs_version = "PY2AND3", srcs_version = "PY2AND3",
deps = [ deps = [
":mobilenet", ":mobilenet",
":mobilenet_common",
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -563,6 +609,7 @@ py_test( # py2and3_test ...@@ -563,6 +609,7 @@ py_test( # py2and3_test
srcs_version = "PY2AND3", srcs_version = "PY2AND3",
deps = [ deps = [
":mobilenet", ":mobilenet",
"//third_party/py/absl/testing:absltest",
# "//tensorflow", # "//tensorflow",
], ],
) )
...@@ -587,6 +634,7 @@ py_test( ...@@ -587,6 +634,7 @@ py_test(
":mobilenet_v1", ":mobilenet_v1",
# "//numpy", # "//numpy",
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -599,6 +647,8 @@ py_binary( ...@@ -599,6 +647,8 @@ py_binary(
":mobilenet_v1", ":mobilenet_v1",
":preprocessing_factory", ":preprocessing_factory",
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/quantize:quantize_graph",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -611,6 +661,8 @@ py_binary( ...@@ -611,6 +661,8 @@ py_binary(
":mobilenet_v1", ":mobilenet_v1",
":preprocessing_factory", ":preprocessing_factory",
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/quantize:quantize_graph",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -620,6 +672,8 @@ py_library( ...@@ -620,6 +672,8 @@ py_library(
srcs_version = "PY2AND3", srcs_version = "PY2AND3",
deps = [ deps = [
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/framework:framework_py",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -630,6 +684,10 @@ py_library( ...@@ -630,6 +684,10 @@ py_library(
deps = [ deps = [
":nasnet_utils", ":nasnet_utils",
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/framework:framework_py",
# "//tensorflow/contrib/layers:layers_py",
# "//tensorflow/contrib/slim",
# "//tensorflow/contrib/training:training_py",
], ],
) )
...@@ -655,6 +713,7 @@ py_test( ...@@ -655,6 +713,7 @@ py_test(
deps = [ deps = [
":nasnet", ":nasnet",
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -666,6 +725,9 @@ py_library( ...@@ -666,6 +725,9 @@ py_library(
":nasnet", ":nasnet",
":nasnet_utils", ":nasnet_utils",
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/framework:framework_py",
# "//tensorflow/contrib/slim",
# "//tensorflow/contrib/training:training_py",
], ],
) )
...@@ -679,6 +741,7 @@ py_test( ...@@ -679,6 +741,7 @@ py_test(
deps = [ deps = [
":pnasnet", ":pnasnet",
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -688,6 +751,7 @@ py_library( ...@@ -688,6 +751,7 @@ py_library(
srcs_version = "PY2AND3", srcs_version = "PY2AND3",
deps = [ deps = [
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -700,6 +764,7 @@ py_test( ...@@ -700,6 +764,7 @@ py_test(
deps = [ deps = [
":overfeat", ":overfeat",
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -709,6 +774,8 @@ py_library( ...@@ -709,6 +774,8 @@ py_library(
srcs_version = "PY2AND3", srcs_version = "PY2AND3",
deps = [ deps = [
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/framework:framework_py",
# "//tensorflow/contrib/layers:layers_py",
], ],
) )
...@@ -720,6 +787,7 @@ py_test( ...@@ -720,6 +787,7 @@ py_test(
deps = [ deps = [
":pix2pix", ":pix2pix",
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/framework:framework_py",
], ],
) )
...@@ -729,6 +797,7 @@ py_library( ...@@ -729,6 +797,7 @@ py_library(
srcs_version = "PY2AND3", srcs_version = "PY2AND3",
deps = [ deps = [
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -739,6 +808,7 @@ py_library( ...@@ -739,6 +808,7 @@ py_library(
deps = [ deps = [
":resnet_utils", ":resnet_utils",
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -755,6 +825,7 @@ py_test( ...@@ -755,6 +825,7 @@ py_test(
":resnet_v1", ":resnet_v1",
# "//numpy", # "//numpy",
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -765,6 +836,7 @@ py_library( ...@@ -765,6 +836,7 @@ py_library(
deps = [ deps = [
":resnet_utils", ":resnet_utils",
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -780,6 +852,7 @@ py_test( ...@@ -780,6 +852,7 @@ py_test(
":resnet_v2", ":resnet_v2",
# "//numpy", # "//numpy",
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -790,6 +863,8 @@ py_library( ...@@ -790,6 +863,8 @@ py_library(
deps = [ deps = [
":i3d_utils", ":i3d_utils",
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/framework:framework_py",
# "//tensorflow/contrib/layers:layers_py",
], ],
) )
...@@ -812,6 +887,7 @@ py_library( ...@@ -812,6 +887,7 @@ py_library(
srcs_version = "PY2AND3", srcs_version = "PY2AND3",
deps = [ deps = [
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -824,6 +900,7 @@ py_test( ...@@ -824,6 +900,7 @@ py_test(
deps = [ deps = [
":vgg", ":vgg",
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -832,7 +909,7 @@ py_library( ...@@ -832,7 +909,7 @@ py_library(
srcs = ["nets/nets_factory.py"], srcs = ["nets/nets_factory.py"],
deps = [ deps = [
":nets", ":nets",
# "//tensorflow", # "//tensorflow/contrib/slim",
], ],
) )
...@@ -858,6 +935,8 @@ py_library( ...@@ -858,6 +935,8 @@ py_library(
":nets_factory", ":nets_factory",
":preprocessing_factory", ":preprocessing_factory",
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/quantize:quantize_graph",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -880,6 +959,8 @@ py_library( ...@@ -880,6 +959,8 @@ py_library(
":nets_factory", ":nets_factory",
":preprocessing_factory", ":preprocessing_factory",
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/quantize:quantize_graph",
# "//tensorflow/contrib/slim",
], ],
) )
...@@ -908,6 +989,8 @@ py_library( ...@@ -908,6 +989,8 @@ py_library(
":dataset_factory", ":dataset_factory",
":nets_factory", ":nets_factory",
# "//tensorflow", # "//tensorflow",
# "//tensorflow/contrib/quantize:quantize_graph",
# "//tensorflow/contrib/slim",
# "//tensorflow/python:platform", # "//tensorflow/python:platform",
], ],
) )
......
research/slim/README.md
View file @ ba87e2c6
...@@ -131,15 +131,11 @@ These represent the training and validation data, sharded over 5 files each. ...@@ -131,15 +131,11 @@ These represent the training and validation data, sharded over 5 files each.
You will also find the `$DATA_DIR/labels.txt` file which contains the mapping You will also find the `$DATA_DIR/labels.txt` file which contains the mapping
from integer labels to class names. from integer labels to class names.
You can use the same script to create the mnist and cifar10 datasets. You can use the same script to create the mnist, cifar10 and visualwakewords
However, for ImageNet, you have to follow the instructions datasets. However, for ImageNet, you have to follow the instructions
[here](https://github.com/tensorflow/models/blob/master/research/inception/README.md#getting-started). [here](https://github.com/tensorflow/models/blob/master/research/inception/README.md#getting-started).
Note that you first have to sign up for an account at image-net.org. Note that you first have to sign up for an account at image-net.org. Also, the
Also, the download can take several hours, and could use up to 500GB. download can take several hours, and could use up to 500GB.
For the visualwakewords dataset, you need to download the MSCOCO dataset [here](https://github.com/tensorflow/models/blob/master/research/slim/datasets/download_mscoco.sh)
and build TFRecords with the following instructions
[here](https://github.com/tensorflow/models/blob/master/research/slim/datasets/build_visualwakewords_data.py).
## Creating a TF-Slim Dataset Descriptor. ## Creating a TF-Slim Dataset Descriptor.
...@@ -149,12 +145,12 @@ which stores pointers to the data file, as well as various other pieces of ...@@ -149,12 +145,12 @@ which stores pointers to the data file, as well as various other pieces of
metadata, such as the class labels, the train/test split, and how to parse the metadata, such as the class labels, the train/test split, and how to parse the
TFExample protos. We have included the TF-Slim Dataset descriptors TFExample protos. We have included the TF-Slim Dataset descriptors
for for
[Cifar10](https://github.com/tensorflow/models/blob/master/research/slim/datasets/cifar10.py),
[ImageNet](https://github.com/tensorflow/models/blob/master/research/slim/datasets/imagenet.py),
[Flowers](https://github.com/tensorflow/models/blob/master/research/slim/datasets/flowers.py), [Flowers](https://github.com/tensorflow/models/blob/master/research/slim/datasets/flowers.py),
[VisualWakeWords](https://github.com/tensorflow/models/blob/master/research/slim/datasets/visualwakewords.py), [Cifar10](https://github.com/tensorflow/models/blob/master/research/slim/datasets/cifar10.py),
[MNIST](https://github.com/tensorflow/models/blob/master/research/slim/datasets/mnist.py),
[ImageNet](https://github.com/tensorflow/models/blob/master/research/slim/datasets/imagenet.py)
and and
[MNIST](https://github.com/tensorflow/models/blob/master/research/slim/datasets/mnist.py). [VisualWakeWords](https://github.com/tensorflow/models/blob/master/research/slim/datasets/visualwakewords.py),
An example of how to load data using a TF-Slim dataset descriptor using a An example of how to load data using a TF-Slim dataset descriptor using a
TF-Slim TF-Slim
[DatasetDataProvider](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/slim/python/slim/data/dataset_data_provider.py) [DatasetDataProvider](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/slim/python/slim/data/dataset_data_provider.py)
......
research/slim/datasets/build_visualwakewords_data.py deleted 100644 → 0
View file @ c2902cfb
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
r"""Build Visual WakeWords Dataset with images and labels for person/not-person.
This script generates the Visual WakeWords dataset annotations from
the raw COCO dataset and converts them to TFRecord.
Visual WakeWords Dataset derives from the COCO dataset to design tiny models
classifying two classes, such as person/not-person. The COCO annotations
are filtered to two classes: foreground_class_of_interest and background
( for e.g. person and not-person). Bounding boxes for small objects
with area less than 5% of the image area are filtered out.
The resulting annotations file has the following fields, where
the image and categories fields are same as COCO dataset, while the annotation
field corresponds to the foreground_class_of_interest/background class and
bounding boxes for the foreground_class_of_interest class.
images{"id", "width", "height", "file_name", "license", "flickr_url",
"coco_url", "date_captured",}
annotations{
"image_id", object[{"category_id", "area", "bbox" : [x,y,width,height],}]
"count",
"label"
}
categories[{
"id", "name", "supercategory",
}]
The TFRecord file contains the following features:
{ image/height, image/width, image/source_id, image/encoded,
image/class/label_text, image/class/label,
image/object/class/text,
image/object/bbox/ymin, image/object/bbox/xmin, image/object/bbox/ymax,
image/object/bbox/xmax, image/object/area
image/filename, image/format, image/key/sha256}
For classification models, you need the image/encoded and image/class/label.
Please note that this tool creates sharded output files.
Example usage:
Add folder tensorflow/models/research/slim to your PYTHONPATH,
and from this folder, run the following commands:
bash download_mscoco.sh path-to-mscoco-dataset
TRAIN_IMAGE_DIR="path-to-mscoco-dataset/train2014"
VAL_IMAGE_DIR="path-to-mscoco-dataset/val2014"
TRAIN_ANNOTATIONS_FILE="path-to-mscoco-dataset/annotations/instances_train2014.json"
VAL_ANNOTATIONS_FILE="path-to-mscoco-dataset/annotations/instances_val2014.json"
python datasets/build_visualwakewords_data.py --logtostderr \
--train_image_dir="${TRAIN_IMAGE_DIR}" \
--val_image_dir="${VAL_IMAGE_DIR}" \
--train_annotations_file="${TRAIN_ANNOTATIONS_FILE}" \
--val_annotations_file="${VAL_ANNOTATIONS_FILE}" \
--output_dir="${OUTPUT_DIR}" \
--small_object_area_threshold=0.005 \
--foreground_class_of_interest='person'
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
import tensorflow as tf
from datasets import build_visualwakewords_data_lib
flags = tf.app.flags
tf.flags.DEFINE_string('train_image_dir', '', 'Training image directory.')
tf.flags.DEFINE_string('val_image_dir', '', 'Validation image directory.')
tf.flags.DEFINE_string('train_annotations_file', '',
'Training annotations JSON file.')
tf.flags.DEFINE_string('val_annotations_file', '',
'Validation annotations JSON file.')
tf.flags.DEFINE_string('output_dir', '/tmp/', 'Output data directory.')
tf.flags.DEFINE_float(
'small_object_area_threshold', 0.005,
'Threshold of fraction of image area below which small'
'objects are filtered')
tf.flags.DEFINE_string(
'foreground_class_of_interest', 'person',
'Build a binary classifier based on the presence or absence'
'of this object in the scene (default is person/not-person)')
FLAGS = flags.FLAGS
tf.logging.set_verbosity(tf.logging.INFO)
def main(unused_argv):
# Path to COCO dataset images and annotations
assert FLAGS.train_image_dir, '`train_image_dir` missing.'
assert FLAGS.val_image_dir, '`val_image_dir` missing.'
assert FLAGS.train_annotations_file, '`train_annotations_file` missing.'
assert FLAGS.val_annotations_file, '`val_annotations_file` missing.'
visualwakewords_annotations_train = os.path.join(
FLAGS.output_dir, 'instances_visualwakewords_train2014.json')
visualwakewords_annotations_val = os.path.join(
FLAGS.output_dir, 'instances_visualwakewords_val2014.json')
visualwakewords_labels_filename = os.path.join(FLAGS.output_dir,
'labels.txt')
small_object_area_threshold = FLAGS.small_object_area_threshold
foreground_class_of_interest = FLAGS.foreground_class_of_interest
# Create the Visual WakeWords annotations from COCO annotations
if not tf.gfile.IsDirectory(FLAGS.output_dir):
tf.gfile.MakeDirs(FLAGS.output_dir)
build_visualwakewords_data_lib.create_visual_wakeword_annotations(
FLAGS.train_annotations_file, visualwakewords_annotations_train,
small_object_area_threshold, foreground_class_of_interest,
visualwakewords_labels_filename)
build_visualwakewords_data_lib.create_visual_wakeword_annotations(
FLAGS.val_annotations_file, visualwakewords_annotations_val,
small_object_area_threshold, foreground_class_of_interest,
visualwakewords_labels_filename)
# Create the TF Records for Visual WakeWords Dataset
if not tf.gfile.IsDirectory(FLAGS.output_dir):
tf.gfile.MakeDirs(FLAGS.output_dir)
train_output_path = os.path.join(FLAGS.output_dir, 'train.record')
val_output_path = os.path.join(FLAGS.output_dir, 'val.record')
build_visualwakewords_data_lib.create_tf_record_for_visualwakewords_dataset(
visualwakewords_annotations_train,
FLAGS.train_image_dir,
train_output_path,
num_shards=100)
build_visualwakewords_data_lib.create_tf_record_for_visualwakewords_dataset(
visualwakewords_annotations_val,
FLAGS.val_image_dir,
val_output_path,
num_shards=10)
if __name__ == '__main__':
tf.app.run()
research/slim/datasets/cifar10.py
View file @ ba87e2c6
...@@ -24,10 +24,11 @@ from __future__ import print_function ...@@ -24,10 +24,11 @@ from __future__ import print_function
import os import os
import tensorflow as tf import tensorflow as tf
from tensorflow.contrib import slim as contrib_slim
from datasets import dataset_utils from datasets import dataset_utils
slim = tf.contrib.slim slim = contrib_slim
_FILE_PATTERN = 'cifar10_%s.tfrecord' _FILE_PATTERN = 'cifar10_%s.tfrecord'
......
research/slim/datasets/dataset_utils.py
View file @ ba87e2c6
...@@ -20,6 +20,7 @@ from __future__ import print_function ...@@ -20,6 +20,7 @@ from __future__ import print_function
import os import os
import sys import sys
import tarfile import tarfile
import zipfile
from six.moves import urllib from six.moves import urllib
import tensorflow as tf import tensorflow as tf
...@@ -101,28 +102,68 @@ def image_to_tfexample(image_data, image_format, height, width, class_id): ...@@ -101,28 +102,68 @@ def image_to_tfexample(image_data, image_format, height, width, class_id):
})) }))
def download_and_uncompress_tarball(tarball_url, dataset_dir): def download_url(url, dataset_dir):
"""Downloads the `tarball_url` and uncompresses it locally. """Downloads the tarball or zip file from url into filepath.
Args: Args:
tarball_url: The URL of a tarball file. url: The URL of a tarball or zip file.
dataset_dir: The directory where the temporary files are stored. dataset_dir: The directory where the temporary files are stored.
Returns:
filepath: path where the file is downloaded.
""" """
filename = tarball_url.split('/')[-1] filename = url.split('/')[-1]
filepath = os.path.join(dataset_dir, filename) filepath = os.path.join(dataset_dir, filename)
def _progress(count, block_size, total_size): def _progress(count, block_size, total_size):
sys.stdout.write('\r>> Downloading %s %.1f%%' % ( sys.stdout.write('\r>> Downloading %s %.1f%%' % (
filename, float(count * block_size) / float(total_size) * 100.0)) filename, float(count * block_size) / float(total_size) * 100.0))
sys.stdout.flush() sys.stdout.flush()
filepath, _ = urllib.request.urlretrieve(tarball_url, filepath, _progress)
filepath, _ = urllib.request.urlretrieve(url, filepath, _progress)
print() print()
statinfo = os.stat(filepath) statinfo = os.stat(filepath)
print('Successfully downloaded', filename, statinfo.st_size, 'bytes.') print('Successfully downloaded', filename, statinfo.st_size, 'bytes.')
return filepath
def download_and_uncompress_tarball(tarball_url, dataset_dir):
"""Downloads the `tarball_url` and uncompresses it locally.
Args:
tarball_url: The URL of a tarball file.
dataset_dir: The directory where the temporary files are stored.
"""
filepath = download_url(tarball_url, dataset_dir)
tarfile.open(filepath, 'r:gz').extractall(dataset_dir) tarfile.open(filepath, 'r:gz').extractall(dataset_dir)
def write_label_file(labels_to_class_names, dataset_dir, def download_and_uncompress_zipfile(zip_url, dataset_dir):
"""Downloads the `zip_url` and uncompresses it locally.
Args:
zip_url: The URL of a zip file.
dataset_dir: The directory where the temporary files are stored.
"""
filename = zip_url.split('/')[-1]
filepath = os.path.join(dataset_dir, filename)
if tf.gfile.Exists(filepath):
print('File {filename} has been already downloaded at {filepath}. '
'Unzipping it....'.format(filename=filename, filepath=filepath))
else:
filepath = download_url(zip_url, dataset_dir)
with zipfile.ZipFile(filepath, 'r') as zip_file:
for member in zip_file.namelist():
memberpath = os.path.join(dataset_dir, member)
# extract only if file doesn't exist
if not (os.path.exists(memberpath) or os.path.isfile(memberpath)):
zip_file.extract(member, dataset_dir)
def write_label_file(labels_to_class_names,
dataset_dir,
filename=LABELS_FILENAME): filename=LABELS_FILENAME):
"""Writes a file with the list of class names. """Writes a file with the list of class names.
......
research/slim/datasets/download_and_convert_visualwakewords.py 0 → 100644
View file @ ba87e2c6
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
r"""Downloads and converts VisualWakewords data to TFRecords of TF-Example protos.
This module downloads the COCO dataset, uncompresses it, derives the
VisualWakeWords dataset to create two TFRecord datasets: one for
train and one for test. Each TFRecord dataset is comprised of a set of
TF-Example protocol buffers, each of which contain a single image and label.
The script should take several minutes to run.
Please note that this tool creates sharded output files.
VisualWakeWords dataset is used to design tiny models classifying two classes,
such as person/not-person. The two steps to generate the VisualWakeWords
dataset from the COCO dataset are given below:
1. Use COCO annotations to create VisualWakeWords annotations:
Note: A bounding box is 'valid' if it has the foreground_class_of_interest
(e.g. person) and it's area is greater than 0.5% of the image area.
The resulting annotations file has the following fields, where 'images' are
the same as COCO dataset. 'categories' only contains information about the
foreground_class_of_interest (e.g. person) and 'annotations' maps an image to
objects (a list of valid bounding boxes) and label (value is 1 if it has
atleast one valid bounding box, otherwise 0)
images[{
"id", "width", "height", "file_name", "flickr_url", "coco_url",
"license", "date_captured",
}]
categories{
"id": {"id", "name", "supercategory"}
}
annotations{
"image_id": {"objects":[{"area", "bbox" : [x,y,width,height]}], "label"}
}
2. Use VisualWakeWords annotations to create TFRecords:
The resulting TFRecord file contains the following features:
{ image/height, image/width, image/source_id, image/encoded,
image/class/label_text, image/class/label,
image/object/class/text,
image/object/bbox/ymin, image/object/bbox/xmin, image/object/bbox/ymax,
image/object/bbox/xmax, image/object/area
image/filename, image/format, image/key/sha256}
For classification models, you need the image/encoded and image/class/label.
Example usage:
Run download_and_convert_data.py in the parent directory as follows:
python download_and_convert_visualwakewords.py --logtostderr \
--dataset_name=visualwakewords \
--dataset_dir="${DATASET_DIR}" \
--small_object_area_threshold=0.005 \
--foreground_class_of_interest='person'
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
import tensorflow as tf
from datasets import download_and_convert_visualwakewords_lib
tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.INFO)
tf.compat.v1.app.flags.DEFINE_string(
'coco_dirname', 'coco_dataset',
'A subdirectory in visualwakewords dataset directory'
'containing the coco dataset')
FLAGS = tf.compat.v1.app.flags.FLAGS
def run(dataset_dir, small_object_area_threshold, foreground_class_of_interest):
"""Runs the download and conversion operation.
Args:
dataset_dir: The dataset directory where the dataset is stored.
small_object_area_threshold: Threshold of fraction of image area below which
small objects are filtered
foreground_class_of_interest: Build a binary classifier based on the
presence or absence of this object in the image.
"""
# 1. Download the coco dataset into a subdirectory under the visualwakewords
# dataset directory
coco_dir = os.path.join(dataset_dir, FLAGS.coco_dirname)
if not tf.gfile.IsDirectory(coco_dir):
tf.gfile.MakeDirs(coco_dir)
download_and_convert_visualwakewords_lib.download_coco_dataset(coco_dir)
# Path to COCO annotations
train_annotations_file = os.path.join(coco_dir, 'annotations',
'instances_train2014.json')
val_annotations_file = os.path.join(coco_dir, 'annotations',
'instances_val2014.json')
train_image_dir = os.path.join(coco_dir, 'train2014')
val_image_dir = os.path.join(coco_dir, 'val2014')
# Path to VisualWakeWords annotations
visualwakewords_annotations_train = os.path.join(
dataset_dir, 'instances_visualwakewords_train2014.json')
visualwakewords_annotations_val = os.path.join(
dataset_dir, 'instances_visualwakewords_val2014.json')
visualwakewords_labels_filename = os.path.join(dataset_dir, 'labels.txt')
train_output_path = os.path.join(dataset_dir, 'train.record')
val_output_path = os.path.join(dataset_dir, 'val.record')
# 2. Create a labels file
tf.logging.info('Creating a labels file...')
download_and_convert_visualwakewords_lib.create_labels_file(
foreground_class_of_interest, visualwakewords_labels_filename)
# 3. Use COCO annotations to create VisualWakeWords annotations
tf.logging.info('Creating train VisualWakeWords annotations...')
download_and_convert_visualwakewords_lib.create_visual_wakeword_annotations(
train_annotations_file, visualwakewords_annotations_train,
small_object_area_threshold, foreground_class_of_interest)
tf.logging.info('Creating validation VisualWakeWords annotations...')
download_and_convert_visualwakewords_lib.create_visual_wakeword_annotations(
val_annotations_file, visualwakewords_annotations_val,
small_object_area_threshold, foreground_class_of_interest)
# 4. Use VisualWakeWords annotations to create the TFRecords
tf.logging.info('Creating train TFRecords for VisualWakeWords dataset...')
download_and_convert_visualwakewords_lib.create_tf_record_for_visualwakewords_dataset(
visualwakewords_annotations_train,
train_image_dir,
train_output_path,
num_shards=100)
tf.logging.info(
'Creating validation TFRecords for VisualWakeWords dataset...')
download_and_convert_visualwakewords_lib.create_tf_record_for_visualwakewords_dataset(
visualwakewords_annotations_val,
val_image_dir,
val_output_path,
num_shards=10)
research/slim/datasets/build_visualwakewords_data_lib.py → research/slim/datasets/download_and_convert_visualwakewords_lib.py
View file @ ba87e2c6
...@@ -12,18 +12,18 @@ ...@@ -12,18 +12,18 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# ============================================================================== # ==============================================================================
r"""Generate Visual Wakewords Dataset. r"""Helper functions to generate the Visual WakeWords dataset.
Helper functions to generate the Visual WakeWords dataset. It filters raw It filters raw COCO annotations file to Visual WakeWords Dataset
COCO annotations file to Visual WakeWords Dataset annotations. annotations. The resulting annotations and COCO images are then converted
The resulting annotations and COCO images are then to TF records.
converted to TF records. See download_and_convert_visualwakewords.py for the sample usage.
See build_visualwakewords_data.py for the sample usage.
""" """
from __future__ import absolute_import from __future__ import absolute_import
from __future__ import division from __future__ import division
from __future__ import print_function from __future__ import print_function
import collections
import hashlib import hashlib
import io import io
import json import json
...@@ -36,31 +36,68 @@ import tensorflow as tf ...@@ -36,31 +36,68 @@ import tensorflow as tf
from datasets import dataset_utils from datasets import dataset_utils
tf.logging.set_verbosity(tf.logging.INFO) tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.INFO)
tf.compat.v1.app.flags.DEFINE_string(
'coco_train_url',
'http://images.cocodataset.org/zips/train2014.zip',
'Link to zip file containing coco training data')
tf.compat.v1.app.flags.DEFINE_string(
'coco_validation_url',
'http://images.cocodataset.org/zips/val2014.zip',
'Link to zip file containing coco validation data')
tf.compat.v1.app.flags.DEFINE_string(
'coco_annotations_url',
'http://images.cocodataset.org/annotations/annotations_trainval2014.zip',
'Link to zip file containing coco annotation data')
FLAGS = tf.compat.v1.app.flags.FLAGS
def download_coco_dataset(dataset_dir):
"""Download the coco dataset.
Args:
dataset_dir: Path where coco dataset should be downloaded.
"""
dataset_utils.download_and_uncompress_zipfile(FLAGS.coco_train_url,
dataset_dir)
dataset_utils.download_and_uncompress_zipfile(FLAGS.coco_validation_url,
dataset_dir)
dataset_utils.download_and_uncompress_zipfile(FLAGS.coco_annotations_url,
dataset_dir)
def create_labels_file(foreground_class_of_interest,
visualwakewords_labels_file):
"""Generate visualwakewords labels file.
Args:
foreground_class_of_interest: category from COCO dataset that is filtered by
the visualwakewords dataset
visualwakewords_labels_file: output visualwakewords label file
"""
labels_to_class_names = {0: 'background', 1: foreground_class_of_interest}
with open(visualwakewords_labels_file, 'w') as fp:
for label in labels_to_class_names:
fp.write(str(label) + ':' + str(labels_to_class_names[label]) + '\n')
def create_visual_wakeword_annotations(annotations_file, def create_visual_wakeword_annotations(annotations_file,
visualwakewords_annotations_path, visualwakewords_annotations_file,
small_object_area_threshold, small_object_area_threshold,
foreground_class_of_interest, foreground_class_of_interest):
visualwakewords_labels_filename):
"""Generate visual wakewords annotations file. """Generate visual wakewords annotations file.
Loads COCO annotation json files and filters to person/not-person Loads COCO annotation json files to generate visualwakewords annotations file.
class (or user-specified class) to generate visual wakewords annotations file.
Each image is assigned a label 1 or 0. The label 1 is assigned as long
as it has at least one foreground_class_of_interest (e.g. person)
bounding box greater than 5% of the image area.
Args: Args:
annotations_file: JSON file containing COCO bounding box annotations annotations_file: JSON file containing COCO bounding box annotations
visualwakewords_annotations_path: output path to annotations file visualwakewords_annotations_file: path to output annotations file
small_object_area_threshold: threshold on fraction of image area below which small_object_area_threshold: threshold on fraction of image area below which
small object bounding boxes are filtered small object bounding boxes are filtered
foreground_class_of_interest: category from COCO dataset that is filtered by foreground_class_of_interest: category from COCO dataset that is filtered by
the visual wakewords dataset the visual wakewords dataset
visualwakewords_labels_filename: The filename to write the visual wakewords
label file
""" """
# default object of interest is person # default object of interest is person
foreground_class_of_interest_id = 1 foreground_class_of_interest_id = 1
...@@ -73,43 +110,28 @@ def create_visual_wakeword_annotations(annotations_file, ...@@ -73,43 +110,28 @@ def create_visual_wakeword_annotations(annotations_file,
if category['name'] == foreground_class_of_interest: if category['name'] == foreground_class_of_interest:
foreground_class_of_interest_id = category['id'] foreground_class_of_interest_id = category['id']
category_index[category['id']] = category category_index[category['id']] = category
# Create annotations index, a map of image_id to it's annotations
# Create annotations index tf.logging.info('Building annotations index...')
annotations_index = {} annotations_index = collections.defaultdict(
annotations_index_filtered = {} lambda: collections.defaultdict(list))
if 'annotations' in groundtruth_data: # structure is { "image_id": {"objects" : [list of the image annotations]}}
tf.logging.info(
'Found groundtruth annotations. Building annotations index.')
for annotation in groundtruth_data['annotations']: for annotation in groundtruth_data['annotations']:
image_id = annotation['image_id'] annotations_index[annotation['image_id']]['objects'].append(annotation)
if image_id not in annotations_index: missing_annotation_count = len(images) - len(annotations_index)
annotations_index[image_id] = []
annotations_index_filtered[image_id] = []
annotations_index[image_id].append(annotation)
missing_annotation_count = 0
for image in images:
image_id = image['id']
if image_id not in annotations_index:
missing_annotation_count += 1
annotations_index[image_id] = []
annotations_index_filtered[image_id] = []
tf.logging.info('%d images are missing annotations.', tf.logging.info('%d images are missing annotations.',
missing_annotation_count) missing_annotation_count)
# Create filtered annotations index # Create filtered annotations index
annotations_index_filtered = {}
for idx, image in enumerate(images): for idx, image in enumerate(images):
if idx % 100 == 0: if idx % 100 == 0:
tf.logging.info('On image %d of %d', idx, len(images)) tf.logging.info('On image %d of %d', idx, len(images))
annotations_list = annotations_index[image['id']] annotations = annotations_index[image['id']]
annotations_list_filtered = _filter_annotations_list( annotations_filtered = _filter_annotations(
annotations_list, image, small_object_area_threshold, annotations, image, small_object_area_threshold,
foreground_class_of_interest_id) foreground_class_of_interest_id)
annotations_index_filtered[image['id']].append(annotations_list_filtered) annotations_index_filtered[image['id']] = annotations_filtered
# Output Visual WakeWords annotations and labels
labels_to_class_names = {0: 'background', 1: foreground_class_of_interest} with open(visualwakewords_annotations_file, 'w') as fp:
with open(visualwakewords_labels_filename, 'w') as fp:
for label in labels_to_class_names:
fp.write(str(label) + ':' + str(labels_to_class_names[label]) + '\n')
with open(visualwakewords_annotations_path, 'w') as fp:
json.dump( json.dump(
{ {
'images': images, 'images': images,
...@@ -118,21 +140,18 @@ def create_visual_wakeword_annotations(annotations_file, ...@@ -118,21 +140,18 @@ def create_visual_wakeword_annotations(annotations_file,
}, fp) }, fp)
def _filter_annotations_list(annotations_list, image, def _filter_annotations(annotations, image, small_object_area_threshold,
small_object_area_threshold,
foreground_class_of_interest_id): foreground_class_of_interest_id):
"""Filters COCO annotations_list to visual wakewords annotations_list. """Filters COCO annotations to visual wakewords annotations.
Each image is assigned a label 1 or 0. The label 1 is assigned as long
as it has at least one foreground_class_of_interest (e.g. person)
bounding box greater than 5% of the image area.
Args: Args:
annotations_list: list of dicts with keys: [ u'id', u'image_id', annotations: dicts with keys: {
u'category_id', u'segmentation', u'area', u'bbox' : [x,y,width,height], u'objects': [{u'id', u'image_id', u'category_id', u'segmentation',
u'iscrowd']. Notice that bounding box coordinates in the official COCO u'area', u'bbox' : [x,y,width,height], u'iscrowd'}] } Notice
dataset are given as [x, y, width, height] tuples using absolute that bounding box coordinates in the official COCO dataset
coordinates where x, y represent the top-left (0-indexed) corner. are given as [x, y, width, height] tuples using absolute
coordinates where x, y represent the top-left (0-indexed)
corner.
image: dict with keys: [u'license', u'file_name', u'coco_url', u'height', image: dict with keys: [u'license', u'file_name', u'coco_url', u'height',
u'width', u'date_captured', u'flickr_url', u'id'] u'width', u'date_captured', u'flickr_url', u'id']
small_object_area_threshold: threshold on fraction of image area below which small_object_area_threshold: threshold on fraction of image area below which
...@@ -141,53 +160,28 @@ def _filter_annotations_list(annotations_list, image, ...@@ -141,53 +160,28 @@ def _filter_annotations_list(annotations_list, image,
wakewords filters wakewords filters
Returns: Returns:
filtered_annotations_list: list of dicts with keys: [ u'image_id', annotations_filtered: dict with keys: {
u'label', u'category_id', u'count', u'objects': [{"area", "bbox" : [x,y,width,height]}],
u'object':[{"category_id", "area", "bbox" : [x,y,width,height],}] u'label',
}
""" """
category_ids = [] objects = []
area = [] image_area = image['height'] * image['width']
flag_small_object = [] for annotation in annotations['objects']:
num_ann = 0 normalized_object_area = annotation['area'] / image_area
image_height = image['height'] category_id = int(annotation['category_id'])
image_width = image['width'] # Filter valid bounding boxes
image_area = image_height * image_width if category_id == foreground_class_of_interest_id and \
bbox = [] normalized_object_area > small_object_area_threshold:
# count of filtered object objects.append({
count = 0 u'area': annotation['area'],
for object_annotations in annotations_list: u'bbox': annotation['bbox'],
(x, y, width, height) = tuple(object_annotations['bbox'])
category_id = int(object_annotations['category_id'])
category_ids.append(category_id)
obj_area = object_annotations['area']
normalized_object_area = obj_area / image_area
# Filter small object bounding boxes
if category_id == foreground_class_of_interest_id:
if normalized_object_area < small_object_area_threshold:
flag_small_object.append(True)
else:
flag_small_object.append(False)
bbox.append({
u'bbox': [x, y, width, height],
u'area': obj_area,
u'category_id': category_id
}) })
count = count + 1 label = 1 if objects else 0
area.append(obj_area)
num_ann = num_ann + 1
# Filtered annotations_list with two classes corresponding to
# foreground_class_of_interest_id (e.g. person) and
# background (e.g. not-person)
if (foreground_class_of_interest_id in category_ids) and (
False in flag_small_object):
return { return {
u'image_id': image['id'], 'objects': objects,
u'label': 1, 'label': label,
u'object': bbox,
u'count': count
} }
else:
return {u'image_id': image['id'], u'label': 0, u'object': [], u'count': 0}
def create_tf_record_for_visualwakewords_dataset(annotations_file, image_dir, def create_tf_record_for_visualwakewords_dataset(annotations_file, image_dir,
...@@ -206,64 +200,35 @@ def create_tf_record_for_visualwakewords_dataset(annotations_file, image_dir, ...@@ -206,64 +200,35 @@ def create_tf_record_for_visualwakewords_dataset(annotations_file, image_dir,
tf_record_close_stack, output_path, num_shards) tf_record_close_stack, output_path, num_shards)
groundtruth_data = json.load(fid) groundtruth_data = json.load(fid)
images = groundtruth_data['images'] images = groundtruth_data['images']
annotations_index = groundtruth_data['annotations']
annotations_index = {int(k): v for k, v in annotations_index.iteritems()}
# convert 'unicode' key to 'int' key after we parse the json file
category_index = {}
for category in groundtruth_data['categories'].values():
# if not background class
if category['id'] != 0:
category_index[category['id']] = category
annotations_index = {}
if 'annotations' in groundtruth_data:
tf.logging.info(
'Found groundtruth annotations. Building annotations index.')
for annotation in groundtruth_data['annotations'].values():
image_id = annotation[0]['image_id']
if image_id not in annotations_index:
annotations_index[image_id] = []
annotations_index[image_id].append(annotation[0])
missing_annotation_count = 0
for image in images:
image_id = image['id']
if image_id not in annotations_index:
missing_annotation_count += 1
annotations_index[image_id] = []
tf.logging.info('%d images are missing annotations.',
missing_annotation_count)
total_num_annotations_skipped = 0
for idx, image in enumerate(images): for idx, image in enumerate(images):
if idx % 100 == 0: if idx % 100 == 0:
tf.logging.info('On image %d of %d', idx, len(images)) tf.logging.info('On image %d of %d', idx, len(images))
annotations_list = annotations_index[image['id']] annotations = annotations_index[image['id']]
_, tf_example, num_annotations_skipped = _create_tf_example( tf_example = _create_tf_example(image, annotations, image_dir)
image, annotations_list[0], image_dir)
total_num_annotations_skipped += num_annotations_skipped
shard_idx = idx % num_shards shard_idx = idx % num_shards
output_tfrecords[shard_idx].write(tf_example.SerializeToString()) output_tfrecords[shard_idx].write(tf_example.SerializeToString())
tf.logging.info('Finished writing, skipped %d annotations.',
total_num_annotations_skipped)
def _create_tf_example(image, annotations_list, image_dir): def _create_tf_example(image, annotations, image_dir):
"""Converts image and annotations to a tf.Example proto. """Converts image and annotations to a tf.Example proto.
Args: Args:
image: dict with keys: [u'license', u'file_name', u'coco_url', u'height', image: dict with keys: [u'license', u'file_name', u'coco_url', u'height',
u'width', u'date_captured', u'flickr_url', u'id'] u'width', u'date_captured', u'flickr_url', u'id']
annotations_list: annotations: dict with objects (a list of image annotations) and a label.
list of dicts with keys: [u'image_id', u'bbox', u'label', {u'objects':[{"area", "bbox" : [x,y,width,height}], u'label'}. Notice
object[{"category_id", "area", "bbox" : [x,y,width,height],}]]. Notice that bounding box coordinates in the COCO dataset are given as[x, y,
that bounding box coordinates in the COCO dataset are given as [x, y,
width, height] tuples using absolute coordinates where x, y represent width, height] tuples using absolute coordinates where x, y represent
the top-left (0-indexed) corner. This function converts to the format the top-left (0-indexed) corner. This function also converts to the format
that can be used by the Tensorflow Object Detection API (which is [ymin, that can be used by the Tensorflow Object Detection API (which is [ymin,
xmin, ymax, xmax] with coordinates normalized relative to image size). xmin, ymax, xmax] with coordinates normalized relative to image size).
image_dir: directory containing the image files. image_dir: directory containing the image files.
Returns: Returns:
example: The converted tf.Example tf_example: The converted tf.Example
num_annotations_skipped: Number of (invalid) annotations that were ignored.
Raises: Raises:
ValueError: if the image pointed to by data['filename'] is not a valid JPEG ValueError: if the image pointed to by data['filename'] is not a valid JPEG
...@@ -280,29 +245,14 @@ def _create_tf_example(image, annotations_list, image_dir): ...@@ -280,29 +245,14 @@ def _create_tf_example(image, annotations_list, image_dir):
image = PIL.Image.open(encoded_jpg_io) image = PIL.Image.open(encoded_jpg_io)
key = hashlib.sha256(encoded_jpg).hexdigest() key = hashlib.sha256(encoded_jpg).hexdigest()
xmin = [] xmin, xmax, ymin, ymax, area = [], [], [], [], []
xmax = [] for obj in annotations['objects']:
ymin = [] (x, y, width, height) = tuple(obj['bbox'])
ymax = []
category_ids = []
area = []
num_annotations_skipped = 0
label = annotations_list['label']
for object_annotations in annotations_list['object']:
(x, y, width, height) = tuple(object_annotations['bbox'])
if width <= 0 or height <= 0:
num_annotations_skipped += 1
continue
if x + width > image_width or y + height > image_height:
num_annotations_skipped += 1
continue
xmin.append(float(x) / image_width) xmin.append(float(x) / image_width)
xmax.append(float(x + width) / image_width) xmax.append(float(x + width) / image_width)
ymin.append(float(y) / image_height) ymin.append(float(y) / image_height)
ymax.append(float(y + height) / image_height) ymax.append(float(y + height) / image_height)
category_id = int(object_annotations['category_id']) area.append(obj['area'])
category_ids.append(category_id)
area.append(object_annotations['area'])
feature_dict = { feature_dict = {
'image/height': 'image/height':
...@@ -320,7 +270,7 @@ def _create_tf_example(image, annotations_list, image_dir): ...@@ -320,7 +270,7 @@ def _create_tf_example(image, annotations_list, image_dir):
'image/format': 'image/format':
dataset_utils.bytes_feature('jpeg'.encode('utf8')), dataset_utils.bytes_feature('jpeg'.encode('utf8')),
'image/class/label': 'image/class/label':
dataset_utils.int64_feature(label), dataset_utils.int64_feature(annotations['label']),
'image/object/bbox/xmin': 'image/object/bbox/xmin':
dataset_utils.float_list_feature(xmin), dataset_utils.float_list_feature(xmin),
'image/object/bbox/xmax': 'image/object/bbox/xmax':
...@@ -329,10 +279,8 @@ def _create_tf_example(image, annotations_list, image_dir): ...@@ -329,10 +279,8 @@ def _create_tf_example(image, annotations_list, image_dir):
dataset_utils.float_list_feature(ymin), dataset_utils.float_list_feature(ymin),
'image/object/bbox/ymax': 'image/object/bbox/ymax':
dataset_utils.float_list_feature(ymax), dataset_utils.float_list_feature(ymax),
'image/object/class/label':
dataset_utils.int64_feature(label),
'image/object/area': 'image/object/area':
dataset_utils.float_list_feature(area), dataset_utils.float_list_feature(area),
} }
example = tf.train.Example(features=tf.train.Features(feature=feature_dict)) example = tf.train.Example(features=tf.train.Features(feature=feature_dict))
return key, example, num_annotations_skipped return example
research/slim/datasets/download_mscoco.sh deleted 100755 → 0
View file @ c2902cfb
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
# Script to download the COCO dataset. See
# http://cocodataset.org/#overview for an overview of the dataset.
#
# usage:
# bash datasets/download_mscoco.sh path-to-COCO-dataset
#
set -e
if [ -z "$1" ]; then
echo "usage download_mscoco.sh [data dir]"
exit
fi
if [ "$(uname)" == "Darwin" ]; then
UNZIP="tar -xf"
else
UNZIP="unzip -nq"
fi
# Create the output directories.
OUTPUT_DIR="${1%/}"
SCRATCH_DIR="${OUTPUT_DIR}/raw-data"
mkdir -p "${OUTPUT_DIR}"
mkdir -p "${SCRATCH_DIR}"
CURRENT_DIR=$(pwd)
# Helper function to download and unpack a .zip file.
function download_and_unzip() {
local BASE_URL=${1}
local FILENAME=${2}
if [ ! -f ${FILENAME} ]; then
echo "Downloading ${FILENAME} to $(pwd)"
wget -nd -c "${BASE_URL}/${FILENAME}"
else
echo "Skipping download of ${FILENAME}"
fi
echo "Unzipping ${FILENAME}"
${UNZIP} ${FILENAME}
}
cd ${SCRATCH_DIR}
# Download the images.
BASE_IMAGE_URL="http://images.cocodataset.org/zips"
TRAIN_IMAGE_FILE="train2014.zip"
download_and_unzip ${BASE_IMAGE_URL} ${TRAIN_IMAGE_FILE}
TRAIN_IMAGE_DIR="${SCRATCH_DIR}/train2014"
VAL_IMAGE_FILE="val2014.zip"
download_and_unzip ${BASE_IMAGE_URL} ${VAL_IMAGE_FILE}
VAL_IMAGE_DIR="${SCRATCH_DIR}/val2014"
# Download the annotations.
BASE_INSTANCES_URL="http://images.cocodataset.org/annotations"
INSTANCES_FILE="annotations_trainval2014.zip"
download_and_unzip ${BASE_INSTANCES_URL} ${INSTANCES_FILE}
research/slim/datasets/flowers.py
View file @ ba87e2c6
...@@ -24,10 +24,11 @@ from __future__ import print_function ...@@ -24,10 +24,11 @@ from __future__ import print_function
import os import os
import tensorflow as tf import tensorflow as tf
from tensorflow.contrib import slim as contrib_slim
from datasets import dataset_utils from datasets import dataset_utils
slim = tf.contrib.slim slim = contrib_slim
_FILE_PATTERN = 'flowers_%s_*.tfrecord' _FILE_PATTERN = 'flowers_%s_*.tfrecord'
......
research/slim/datasets/imagenet.py
View file @ ba87e2c6
...@@ -35,10 +35,11 @@ from __future__ import print_function ...@@ -35,10 +35,11 @@ from __future__ import print_function
import os import os
from six.moves import urllib from six.moves import urllib
import tensorflow as tf import tensorflow as tf
from tensorflow.contrib import slim as contrib_slim
from datasets import dataset_utils from datasets import dataset_utils
slim = tf.contrib.slim slim = contrib_slim
# TODO(nsilberman): Add tfrecord file type once the script is updated. # TODO(nsilberman): Add tfrecord file type once the script is updated.
_FILE_PATTERN = '%s-*' _FILE_PATTERN = '%s-*'
......
research/slim/datasets/mnist.py
View file @ ba87e2c6
...@@ -24,10 +24,11 @@ from __future__ import print_function ...@@ -24,10 +24,11 @@ from __future__ import print_function
import os import os
import tensorflow as tf import tensorflow as tf
from tensorflow.contrib import slim as contrib_slim
from datasets import dataset_utils from datasets import dataset_utils
slim = tf.contrib.slim slim = contrib_slim
_FILE_PATTERN = 'mnist_%s.tfrecord' _FILE_PATTERN = 'mnist_%s.tfrecord'
......
research/slim/datasets/visualwakewords.py
View file @ ba87e2c6
...@@ -29,11 +29,12 @@ from __future__ import print_function ...@@ -29,11 +29,12 @@ from __future__ import print_function
import os import os
import tensorflow as tf import tensorflow as tf
from tensorflow.contrib import slim as contrib_slim
from datasets import dataset_utils from datasets import dataset_utils
slim = tf.contrib.slim slim = contrib_slim
_FILE_PATTERN = '%s.record-*' _FILE_PATTERN = '%s.record-*'
...@@ -47,7 +48,6 @@ _ITEMS_TO_DESCRIPTIONS = { ...@@ -47,7 +48,6 @@ _ITEMS_TO_DESCRIPTIONS = {
'image': 'A color image of varying height and width.', 'image': 'A color image of varying height and width.',
'label': 'The label id of the image, an integer in {0, 1}', 'label': 'The label id of the image, an integer in {0, 1}',
'object/bbox': 'A list of bounding boxes.', 'object/bbox': 'A list of bounding boxes.',
'object/label': 'A list of labels, all objects belong to the same class.',
} }
_NUM_CLASSES = 2 _NUM_CLASSES = 2
...@@ -99,8 +99,6 @@ def get_split(split_name, dataset_dir, file_pattern=None, reader=None): ...@@ -99,8 +99,6 @@ def get_split(split_name, dataset_dir, file_pattern=None, reader=None):
tf.VarLenFeature(dtype=tf.float32), tf.VarLenFeature(dtype=tf.float32),
'image/object/bbox/ymax': 'image/object/bbox/ymax':
tf.VarLenFeature(dtype=tf.float32), tf.VarLenFeature(dtype=tf.float32),
'image/object/class/label':
tf.VarLenFeature(dtype=tf.int64),
} }
items_to_handlers = { items_to_handlers = {
...@@ -111,8 +109,6 @@ def get_split(split_name, dataset_dir, file_pattern=None, reader=None): ...@@ -111,8 +109,6 @@ def get_split(split_name, dataset_dir, file_pattern=None, reader=None):
'object/bbox': 'object/bbox':
slim.tfexample_decoder.BoundingBox(['ymin', 'xmin', 'ymax', 'xmax'], slim.tfexample_decoder.BoundingBox(['ymin', 'xmin', 'ymax', 'xmax'],
'image/object/bbox/'), 'image/object/bbox/'),
'object/label':
slim.tfexample_decoder.Tensor('image/object/class/label'),
} }
decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features, decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features,
......
research/slim/deployment/model_deploy.py
View file @ ba87e2c6
...@@ -102,8 +102,9 @@ from __future__ import print_function ...@@ -102,8 +102,9 @@ from __future__ import print_function
import collections import collections
import tensorflow as tf import tensorflow as tf
from tensorflow.contrib import slim as contrib_slim
slim = tf.contrib.slim slim = contrib_slim
__all__ = ['create_clones', __all__ = ['create_clones',
......
research/slim/deployment/model_deploy_test.py
View file @ ba87e2c6
...@@ -20,10 +20,13 @@ from __future__ import print_function ...@@ -20,10 +20,13 @@ from __future__ import print_function
import numpy as np import numpy as np
import tensorflow as tf import tensorflow as tf
from tensorflow.contrib import framework as contrib_framework
from tensorflow.contrib import layers as contrib_layers
from tensorflow.contrib import slim as contrib_slim
from deployment import model_deploy from deployment import model_deploy
slim = tf.contrib.slim slim = contrib_slim
class DeploymentConfigTest(tf.test.TestCase): class DeploymentConfigTest(tf.test.TestCase):
...@@ -508,9 +511,8 @@ class DeployTest(tf.test.TestCase): ...@@ -508,9 +511,8 @@ class DeployTest(tf.test.TestCase):
with tf.Session() as sess: with tf.Session() as sess:
sess.run(tf.global_variables_initializer()) sess.run(tf.global_variables_initializer())
moving_mean = tf.contrib.framework.get_variables_by_name( moving_mean = contrib_framework.get_variables_by_name('moving_mean')[0]
'moving_mean')[0] moving_variance = contrib_framework.get_variables_by_name(
moving_variance = tf.contrib.framework.get_variables_by_name(
'moving_variance')[0] 'moving_variance')[0]
initial_loss = sess.run(model.total_loss) initial_loss = sess.run(model.total_loss)
initial_mean, initial_variance = sess.run([moving_mean, initial_mean, initial_variance = sess.run([moving_mean,
...@@ -537,8 +539,8 @@ class DeployTest(tf.test.TestCase): ...@@ -537,8 +539,8 @@ class DeployTest(tf.test.TestCase):
# clone function creates a fully_connected layer with a regularizer loss. # clone function creates a fully_connected layer with a regularizer loss.
def ModelFn(): def ModelFn():
inputs = tf.constant(1.0, shape=(10, 20), dtype=tf.float32) inputs = tf.constant(1.0, shape=(10, 20), dtype=tf.float32)
reg = tf.contrib.layers.l2_regularizer(0.001) reg = contrib_layers.l2_regularizer(0.001)
tf.contrib.layers.fully_connected(inputs, 30, weights_regularizer=reg) contrib_layers.fully_connected(inputs, 30, weights_regularizer=reg)
model = model_deploy.deploy( model = model_deploy.deploy(
deploy_config, ModelFn, deploy_config, ModelFn,
...@@ -556,8 +558,8 @@ class DeployTest(tf.test.TestCase): ...@@ -556,8 +558,8 @@ class DeployTest(tf.test.TestCase):
# clone function creates a fully_connected layer with a regularizer loss. # clone function creates a fully_connected layer with a regularizer loss.
def ModelFn(): def ModelFn():
inputs = tf.constant(1.0, shape=(10, 20), dtype=tf.float32) inputs = tf.constant(1.0, shape=(10, 20), dtype=tf.float32)
reg = tf.contrib.layers.l2_regularizer(0.001) reg = contrib_layers.l2_regularizer(0.001)
tf.contrib.layers.fully_connected(inputs, 30, weights_regularizer=reg) contrib_layers.fully_connected(inputs, 30, weights_regularizer=reg)
# No optimizer here, it's an eval. # No optimizer here, it's an eval.
model = model_deploy.deploy(deploy_config, ModelFn) model = model_deploy.deploy(deploy_config, ModelFn)
......
research/slim/download_and_convert_data.py
View file @ ba87e2c6
...@@ -18,16 +18,21 @@ Usage: ...@@ -18,16 +18,21 @@ Usage:
```shell ```shell
$ python download_and_convert_data.py \ $ python download_and_convert_data.py \
--dataset_name=mnist \ --dataset_name=flowers \
--dataset_dir=/tmp/mnist --dataset_dir=/tmp/flowers
$ python download_and_convert_data.py \ $ python download_and_convert_data.py \
--dataset_name=cifar10 \ --dataset_name=cifar10 \
--dataset_dir=/tmp/cifar10 --dataset_dir=/tmp/cifar10
$ python download_and_convert_data.py \ $ python download_and_convert_data.py \
--dataset_name=flowers \ --dataset_name=mnist \
--dataset_dir=/tmp/flowers --dataset_dir=/tmp/mnist
$ python download_and_convert_data.py \
--dataset_name=visualwakewords \
--dataset_dir=/tmp/visualwakewords
``` ```
""" """
from __future__ import absolute_import from __future__ import absolute_import
...@@ -39,19 +44,31 @@ import tensorflow as tf ...@@ -39,19 +44,31 @@ import tensorflow as tf
from datasets import download_and_convert_cifar10 from datasets import download_and_convert_cifar10
from datasets import download_and_convert_flowers from datasets import download_and_convert_flowers
from datasets import download_and_convert_mnist from datasets import download_and_convert_mnist
from datasets import download_and_convert_visualwakewords
FLAGS = tf.app.flags.FLAGS FLAGS = tf.compat.v1.app.flags.FLAGS
tf.app.flags.DEFINE_string( tf.compat.v1.app.flags.DEFINE_string(
'dataset_name', 'dataset_name',
None, None,
'The name of the dataset to convert, one of "cifar10", "flowers", "mnist".') 'The name of the dataset to convert, one of "flowers", "cifar10", "mnist", "visualwakewords"'
)
tf.app.flags.DEFINE_string( tf.compat.v1.app.flags.DEFINE_string(
'dataset_dir', 'dataset_dir',
None, None,
'The directory where the output TFRecords and temporary files are saved.') 'The directory where the output TFRecords and temporary files are saved.')
tf.flags.DEFINE_float(
'small_object_area_threshold', 0.005,
'For --dataset_name=visualwakewords only. Threshold of fraction of image '
'area below which small objects are filtered')
tf.flags.DEFINE_string(
'foreground_class_of_interest', 'person',
'For --dataset_name=visualwakewords only. Build a binary classifier based '
'on the presence or absence of this object in the image.')
def main(_): def main(_):
if not FLAGS.dataset_name: if not FLAGS.dataset_name:
...@@ -59,15 +76,19 @@ def main(_): ...@@ -59,15 +76,19 @@ def main(_):
if not FLAGS.dataset_dir: if not FLAGS.dataset_dir:
raise ValueError('You must supply the dataset directory with --dataset_dir') raise ValueError('You must supply the dataset directory with --dataset_dir')
if FLAGS.dataset_name == 'cifar10': if FLAGS.dataset_name == 'flowers':
download_and_convert_cifar10.run(FLAGS.dataset_dir)
elif FLAGS.dataset_name == 'flowers':
download_and_convert_flowers.run(FLAGS.dataset_dir) download_and_convert_flowers.run(FLAGS.dataset_dir)
elif FLAGS.dataset_name == 'cifar10':
download_and_convert_cifar10.run(FLAGS.dataset_dir)
elif FLAGS.dataset_name == 'mnist': elif FLAGS.dataset_name == 'mnist':
download_and_convert_mnist.run(FLAGS.dataset_dir) download_and_convert_mnist.run(FLAGS.dataset_dir)
elif FLAGS.dataset_name == 'visualwakewords':
download_and_convert_visualwakewords.run(
FLAGS.dataset_dir, FLAGS.small_object_area_threshold,
FLAGS.foreground_class_of_interest)
else: else:
raise ValueError( raise ValueError(
'dataset_name [%s] was not recognized.' % FLAGS.dataset_name) 'dataset_name [%s] was not recognized.' % FLAGS.dataset_name)
if __name__ == '__main__': if __name__ == '__main__':
tf.app.run() tf.compat.v1.app.run()
research/slim/eval_image_classifier.py
View file @ ba87e2c6
...@@ -20,12 +20,14 @@ from __future__ import print_function ...@@ -20,12 +20,14 @@ from __future__ import print_function
import math import math
import tensorflow as tf import tensorflow as tf
from tensorflow.contrib import quantize as contrib_quantize
from tensorflow.contrib import slim as contrib_slim
from datasets import dataset_factory from datasets import dataset_factory
from nets import nets_factory from nets import nets_factory
from preprocessing import preprocessing_factory from preprocessing import preprocessing_factory
slim = tf.contrib.slim slim = contrib_slim
tf.app.flags.DEFINE_integer( tf.app.flags.DEFINE_integer(
'batch_size', 100, 'The number of samples in each batch.') 'batch_size', 100, 'The number of samples in each batch.')
...@@ -146,7 +148,7 @@ def main(_): ...@@ -146,7 +148,7 @@ def main(_):
logits, _ = network_fn(images) logits, _ = network_fn(images)
if FLAGS.quantize: if FLAGS.quantize:
tf.contrib.quantize.create_eval_graph() contrib_quantize.create_eval_graph()
if FLAGS.moving_average_decay: if FLAGS.moving_average_decay:
variable_averages = tf.train.ExponentialMovingAverage( variable_averages = tf.train.ExponentialMovingAverage(
......
research/slim/export_inference_graph.py
View file @ ba87e2c6
...@@ -58,13 +58,15 @@ from __future__ import print_function ...@@ -58,13 +58,15 @@ from __future__ import print_function
import os import os
import tensorflow as tf import tensorflow as tf
from tensorflow.contrib import quantize as contrib_quantize
from tensorflow.contrib import slim as contrib_slim
from tensorflow.python.platform import gfile from tensorflow.python.platform import gfile
from datasets import dataset_factory from datasets import dataset_factory
from nets import nets_factory from nets import nets_factory
slim = tf.contrib.slim slim = contrib_slim
tf.app.flags.DEFINE_string( tf.app.flags.DEFINE_string(
'model_name', 'inception_v3', 'The name of the architecture to save.') 'model_name', 'inception_v3', 'The name of the architecture to save.')
...@@ -144,7 +146,7 @@ def main(_): ...@@ -144,7 +146,7 @@ def main(_):
network_fn(placeholder) network_fn(placeholder)
if FLAGS.quantize: if FLAGS.quantize:
tf.contrib.quantize.create_eval_graph() contrib_quantize.create_eval_graph()
graph_def = graph.as_graph_def() graph_def = graph.as_graph_def()
if FLAGS.write_text_graphdef: if FLAGS.write_text_graphdef:
......
research/slim/nets/alexnet.py
View file @ ba87e2c6
...@@ -37,8 +37,9 @@ from __future__ import division ...@@ -37,8 +37,9 @@ from __future__ import division
from __future__ import print_function from __future__ import print_function
import tensorflow as tf import tensorflow as tf
from tensorflow.contrib import slim as contrib_slim
slim = tf.contrib.slim slim = contrib_slim
trunc_normal = lambda stddev: tf.truncated_normal_initializer(0.0, stddev) trunc_normal = lambda stddev: tf.truncated_normal_initializer(0.0, stddev)
......
research/slim/nets/alexnet_test.py
View file @ ba87e2c6
...@@ -18,10 +18,11 @@ from __future__ import division ...@@ -18,10 +18,11 @@ from __future__ import division
from __future__ import print_function from __future__ import print_function
import tensorflow as tf import tensorflow as tf
from tensorflow.contrib import slim as contrib_slim
from nets import alexnet from nets import alexnet
slim = tf.contrib.slim slim = contrib_slim
class AlexnetV2Test(tf.test.TestCase): class AlexnetV2Test(tf.test.TestCase):
......
research/slim/nets/cifarnet.py
View file @ ba87e2c6
...@@ -19,8 +19,9 @@ from __future__ import division ...@@ -19,8 +19,9 @@ from __future__ import division
from __future__ import print_function from __future__ import print_function
import tensorflow as tf import tensorflow as tf
from tensorflow.contrib import slim as contrib_slim
slim = tf.contrib.slim slim = contrib_slim
trunc_normal = lambda stddev: tf.truncated_normal_initializer(stddev=stddev) trunc_normal = lambda stddev: tf.truncated_normal_initializer(stddev=stddev)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment